tikv
/
client-java
mirror of https://github.com/tikv/client-java.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Initial java client for TiKV

This commit is contained in:
Huachao Huang 2018-11-15 20:09:15 +08:00 committed by GitHub
parent 50be50807d fdf6ee3b06
commit 47af16be9e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
191 changed files with 24838 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
.gitignore vendored Normal file
View File

@ -0,0 +1,59 @@
#ignore idea configuration
.idea
*.iml
pub.sh
# ignore compiled classes
target
# ignore version info
src/main/java/com/pingcap/tikv/TiVersion.java
# ignore eclipse configuration
.project
# ignore maven related files
dependency-reduced-pom.xml
# ignore emacs configuation
.meghanada
# Compiled class file
*.class
# Log file
*.log
# BlueJ files
*.ctxt
# Mobile Tools for Java (J2ME)
.mtj.tmp/
# Package Files #
*.jar
*.war
*.ear
*.zip
*.tar.gz
*.rar
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
# ignore Mac files
.DS_Store
# ignore gradle
.gradle
metastore_db/
.java-version
# ignore tikv-client proto files
kvproto
tipb
proto
raft-rs
# ignore tikv-client bazel build
bazel-*

49
BUILD Normal file
View File

@ -0,0 +1,49 @@
load(
"@com_github_zhexuany_bazel_shade//:java_shade.bzl",
"java_shade"
)
package(default_visibility = ["//visibility:public"])
java_binary(
name = "tikv-java-client",
main_class = "com.pingcap.tikv.Main",
runtime_deps = [
"//src/main/java/com/pingcap/tikv:tikv-java-client-lib",
":shaded_scalding",
],
)
java_shade(
name = "shaded_args",
input_jar = "@io_netty_netty_codec_socks//jar",
rules = "shading_rule"
)
java_import(
name = "shaded_scalding",
jars = ["shaded_args.jar"]
)
filegroup(
name = "protos",
srcs = glob([
"kvproto/proto/*.proto",
"kvproto/_vendor/src/github.com/gogo/protobuf/gogoproto/*.proto",
"tipb/proto/*.proto",
]),
)
load("@org_pubref_rules_protobuf//java:rules.bzl", "java_proto_library")
java_proto_library(
name = "java",
imports = [
"external/com_google_protobuf/src/",
"kvproto/proto",
"kvproto/_vendor/src/github.com/gogo/protobuf",
"tipb/proto",
],
inputs = ["@com_google_protobuf//:well_known_protos"],
protos = [":protos"],
verbose = 0, # 0=no output, 1=show protoc command, 2+ more...
with_grpc = True,
)

4
LICENSE
View File

@ -178,7 +178,7 @@
APPENDIX: How to apply the Apache License to your work. APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]" boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a comment syntax for the file format. We also recommend that a
@ -186,7 +186,7 @@
same "printed page" as the copyright notice for easier same "printed page" as the copyright notice for easier
identification within third-party archives. identification within third-party archives.
Copyright [yyyy] [name of copyright owner] Copyright {2017} {PingCap.Inc}
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.

6
Makefile Normal file
View File

@ -0,0 +1,6 @@
run:
bazel run :tikv-java-client
uber_jar:
bazel build :tikv-java-client_deploy.jar
test:
bazel test //src/test/java/com/pingcap/tikv:tikv-client-java-test --test_output=errors --test_timeout=3600

189
README.md Normal file
View File

@ -0,0 +1,189 @@
## Ti-Client in Java [Under Construction]
A Java client for [TiDB](https://github.com/pingcap/tidb)/[TiKV](https://github.com/pingcap/tikv).
It is supposed to:
+ Communicate via [gRPC](http://www.grpc.io/)
+ Talk to Placement Driver searching for a region
+ Talk to TiKV for reading/writing data and the resulted data is encoded/decoded just like what we do in TiDB.
+ Talk to Coprocessor for calculation pushdown
## How to build
The alternative way to build a usable jar for testing will be
```
mvn clean install -Dmaven.test.skip=true
```
The following command can install dependencies for you.
```
mvn package
```
Alternatively, you can use `bazel` for much faster build. When you try this approach, you should run `git submodule update --init --recursive` before you build project.
Making a uber jar:
```
make uber_jar
```
run Main class:
```
make run
```
run test cases:
```
make test
```
this project is designed to hook with `pd` and `tikv` which you can find in `PingCap` github page.
When you work with this project, you have to communicate with `pd` and `tikv`. There is a script taking care of this. By executing the following commands, `pd` and `tikv` can be executed on background.
```
cd scripts
make pd
make tikv
```
## How to use for now
Since it's not quite complete, a usage sample for now can be given is:
```java
// Init tidb cluster configuration
TiConfiguration conf = TiConfiguration.createDefault("127.0.0.1:2379");
TiSession session = TiSession.create(conf);
Catalog cat = session.getCatalog();
TiDBInfo db = cat.getDatabase("tpch_test");
TiTableInfo table = cat.getTable(db, "customer");
Snapshot snapshot = session.createSnapshot();
// Generate select ranges
ByteString startKey = TableCodec.encodeRowKeyWithHandle(table.getId(), Long.MIN_VALUE);
ByteString endKey = TableCodec.encodeRowKeyWithHandle(table.getId(), Long.MAX_VALUE);
Coprocessor.KeyRange keyRange = Coprocessor.KeyRange.newBuilder().setStart(startKey).setEnd(endKey).build();
List<Coprocessor.KeyRange> ranges = new ArrayList<>();
ranges.add(keyRange);
// Create select request
TiSelectRequest selectRequest = new TiSelectRequest();
selectRequest.addRanges(ranges);
selectRequest.addField(TiColumnRef.create("c_mktsegment", table));
selectRequest.setTableInfo(table);
selectRequest.setStartTs(session.getTimestamp().getVersion());
selectRequest.addWhere(new GreaterEqual(TiConstant.create(5), TiConstant.create(5)));
selectRequest.addGroupByItem(TiByItem.create(TiColumnRef.create("c_mktsegment"), false));
selectRequest.setLimit(10);
selectRequest.bind();
// Fetch data
Iterator<Row> iterator = snapshot.select(selectRequest);
System.out.println("Show result:");
while (iterator.hasNext()) {
Row rowData = iterator.next();
for (int i = 0; i < rowData.fieldCount(); i++) {
System.out.print(rowData.get(i, null) + "\t");
}
System.out.println();
}
```
Result:
```java
Show result:
BUILDING
AUTOMOBILE
MACHINERY
HOUSEHOLD
FURNITURE
```
## Raw TiKV-Client in Java
Java Implementation of Raw TiKV-Client
Demo is avaliable in [KVRawClientTest](https://github.com/birdstorm/KVRawClientTest/)
### Build
```
mvn clean install -Dmaven.test.skip=true
```
### Use as maven dependency
After building, add following lines into your `pom.xml`
```xml
<dependency>
<groupId>com.pingcap.tikv</groupId>
<artifactId>tikv-client-java</artifactId>
<version>2.0-SNAPSHOT</version>
</dependency>
```
### Entrance
`com.pingcap.tikv.RawKVClient`
### API
```java
/**
* create a RawKVClient using specific pd addresses
*
* @param address pd addresses(comma seperated)
*/
static RawKVClient create(String address)
```
```java
/**
* Put a raw key-value pair to TiKV
*
* @param key raw key
* @param value raw value
*/
void put(ByteString key, ByteString value)
```
```java
/**
* Get a raw key-value pair from TiKV if key exists
*
* @param key raw key
* @return a ByteString value if key exists, ByteString.EMPTY if key does not exist
*/
ByteString get(ByteString key)
```
```java
/**
* Scan raw key-value pairs from TiKV in range [startKey, endKey)
*
* @param startKey raw start key, inclusive
* @param endKey raw end key, exclusive
* @return list of key-value pairs in range
*/
List<Kvrpcpb.KvPair> scan(ByteString startKey, ByteString endKey)
```
```java
/**
* Scan raw key-value pairs from TiKV in range [startKey, endKey)
*
* @param startKey raw start key, inclusive
* @param limit limit of key-value pairs
* @return list of key-value pairs in range
*/
List<Kvrpcpb.KvPair> scan(ByteString startKey, int limit)
```
```java
/**
* Delete a raw key-value pair from TiKV if key exists
*
* @param key raw key to be deleted
*/
void delete(ByteString key)
```
## TODO
Contributions are welcomed. Here is a [TODO](https://github.com/pingcap/tikv-client-java/wiki/TODO-Lists) and you might contact maxiaoyu@pingcap.com if needed.
## License
Apache 2.0 license. See the [LICENSE](./LICENSE) file for details.

112
WORKSPACE Normal file
View File

@ -0,0 +1,112 @@
maven_jar(
name = "com_fasterxml_jackson_core_jackson_annotations",
artifact = "com.fasterxml.jackson.core:jackson-annotations:2.6.6",
)
maven_jar(
name = "com_fasterxml_jackson_core_jackson_databind",
artifact = "com.fasterxml.jackson.core:jackson-databind:2.6.6",
)
maven_jar(
name = "com_fasterxml_jackson_core_jackson_core",
artifact = "com.fasterxml.jackson.core:jackson-core:2.6.6",
)
maven_jar(
name = "org_slf4j_slf4j_api",
artifact = "org.slf4j:slf4j-api:1.7.16",
)
maven_jar(
name = "org_slf4j_jcl_over_slf4j",
artifact = "org.slf4j:jcl-over-slf4j:1.7.16",
)
maven_jar(
name = "org_slf4j_jul_to_slf4j",
artifact = "org.slf4j:jul-to-slf4j:1.7.16",
)
maven_jar(
name = "log4j_log4j",
artifact = "log4j:log4j:1.2.17",
)
maven_jar(
name = "joda_time",
artifact = "joda-time:joda-time:2.9.9",
)
maven_jar(
name = "junit_junit",
artifact = "junit:junit:4.12",
)
maven_jar(
name = "org_hamcrest_hamcrest_core",
artifact = "org.hamcrest:hamcrest-core:1.3",
)
maven_jar(
name = "org_javassist_javassist",
artifact = "org.javassist:javassist:3.21.0-GA",
)
maven_jar(
name = "org_powermock_powermock_reflect",
artifact = "org.powermock:powermock-reflect:1.6.6",
)
maven_jar(
name = "org_powermock_powermock_api_mockito",
artifact = "org.powermock:powermock-api-mockito:1.6.6",
)
maven_jar(
name = "org_mockito_mockito_core",
artifact = "org.mockito:mockito-core:1.10.19",
)
maven_jar(
name = "org_objenesis_objenesis",
artifact = "org.objenesis:objenesis:2.1",
)
maven_jar(
name = "org_powermock_powermock_api_mockito_common",
artifact = "org.powermock:powermock-api-mockito-common:1.6.6",
)
maven_jar(
name = "org_powermock_powermock_api_support",
artifact = "org.powermock:powermock-api-support:1.6.6",
)
maven_jar(
name = "net_sf_trove4j_trove4j",
artifact = "net.sf.trove4j:trove4j:3.0.1",
)
git_repository(
name = "org_pubref_rules_protobuf",
remote = "https://github.com/pubref/rules_protobuf",
tag = "v0.8.1",
)
load("@org_pubref_rules_protobuf//java:rules.bzl", "java_proto_repositories")
java_proto_repositories()
bazel_shade_version = "master"
http_archive(
name = "com_github_zhexuany_bazel_shade",
url = "https://github.com/zhexuany/bazel_shade_plugin/archive/%s.zip"%bazel_shade_version,
type = "zip",
strip_prefix= "bazel_shade_plugin-%s"%bazel_shade_version
)
load(
"@com_github_zhexuany_bazel_shade//:java_shade.bzl",
"java_shade_repositories",
"java_shade"
)
java_shade_repositories()

458
pom.xml Normal file
View File

@ -0,0 +1,458 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.pingcap.tikv</groupId>
<artifactId>tikv-client-java</artifactId>
<version>2.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>TiSpark Project TiKV Java Client</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<protobuf.version>3.1.0</protobuf.version>
<junit.version>4.12</junit.version>
<log4j.version>1.2.17</log4j.version>
<slf4j.version>1.7.16</slf4j.version>
<grpc.version>1.7.0</grpc.version>
<powermock.version>1.6.6</powermock.version>
<jackson.version>2.8.11</jackson.version>
<jackson.databind.version>2.8.11.1</jackson.databind.version>
<trove4j.version>3.0.1</trove4j.version>
<joda-time.version>2.9.9</joda-time.version>
<joda-convert.version>1.9.2</joda-convert.version>
<spark.version>2.3.2</spark.version>
<proto.folder>${basedir}/proto</proto.folder>
<scala.binary.version>2.11</scala.binary.version>
<scala.version>2.11</scala.version>
<scalatest.version>3.0.4</scalatest.version>
<gpg.skip>true</gpg.skip>
<javadoc.skip>true</javadoc.skip>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.8.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.8.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-unsafe_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jul-to-slf4j</artifactId>
<version>${slf4j.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
<version>${slf4j.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>${log4j.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/net.sf.trove4j/trove4j -->
<dependency>
<groupId>net.sf.trove4j</groupId>
<artifactId>trove4j</artifactId>
<version>${trove4j.version}</version>
</dependency>
<dependency>
<groupId>com.sangupta</groupId>
<artifactId>murmur</artifactId>
<version>1.0.0</version>
</dependency>
<!-- grpc dependencies -->
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.databind.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
<version>${jackson.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>${joda-time.version}</version>
</dependency>
<dependency>
<groupId>org.joda</groupId>
<artifactId>joda-convert</artifactId>
<version>${joda-convert.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-testing</artifactId>
<version>${grpc.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>1.4.1.Final</version>
</extension>
</extensions>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.6.0</version>
<executions>
<execution>
<id>clone proto files</id>
<configuration>
<executable>${basedir}/scripts/proto.sh</executable>
</configuration>
<phase>validate</phase>
<goals>
<goal>exec</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>2.5</version>
<executions>
<execution>
<id>copy-resources</id>
<phase>validate</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>${proto.folder}</outputDirectory>
<resources>
<resource>
<directory>${basedir}/kvproto/_vendor/src/github.com/gogo/protobuf</directory>
<includes>
<include>**/gogoproto/**</include>
</includes>
</resource>
<resource>
<directory>${basedir}/kvproto/proto</directory>
<filtering>true</filtering>
</resource>
<resource>
<directory>${basedir}/raft-rs/proto</directory>
<filtering>true</filtering>
</resource>
<resource>
<directory>${basedir}/tipb/proto</directory>
<filtering>true</filtering>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<version>0.5.0</version>
<configuration>
<protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
<protoSourceRoot>${proto.folder}</protoSourceRoot>
<includes>
<param>**/*.proto</param>
</includes>
<pluginId>grpc-java</pluginId>
<pluginArtifact>io.grpc:protoc-gen-grpc-java:1.4.0:exe:${os.detected.classifier}</pluginArtifact>
</configuration>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>compile-custom</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Compiler Plug-in -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
<showWarnings>true</showWarnings>
<showDeprecation>true</showDeprecation>
</configuration>
</plugin>
<!-- Jar Plug-in -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
<configuration>
<archive>
<addMavenDescriptor>false</addMavenDescriptor>
</archive>
</configuration>
</plugin>
<!-- Source Plug-in -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>3.0.1</version>
<executions>
<execution>
<id>attach-sources</id>
<phase>deploy</phase>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Clean Plug-in -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-clean-plugin</artifactId>
<version>3.0.0</version>
<configuration>
<filesets>
<fileset>
<directory>${proto.folder}</directory>
</fileset>
</filesets>
</configuration>
</plugin>
<!-- Javadoc Plug-in -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<configuration>
<skip>${javadoc.skip}</skip>
</configuration>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
<configuration> <!-- add this to disable checking -->
<additionalparam>-Xdoclint:none</additionalparam>
</configuration>
</execution>
</executions>
</plugin>
<!--GPG Signed Components-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>1.5</version>
<configuration>
<keyname>Yifei Wu</keyname>
<skip>${gpg.skip}</skip>
</configuration>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Assembly Plug-in -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>assemble-all</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<!--- Needs to shade Protobuf 3 since other projects might use other version -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
<relocations>
<relocation>
<pattern>com.fasterxml</pattern>
<shadedPattern>shade.com.fasterxml.jackson</shadedPattern>
</relocation>
<relocation>
<pattern>io.grpc</pattern>
<shadedPattern>shade.io.grpc</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.protobuf</pattern>
<shadedPattern>shade.com.google.protobuf</shadedPattern>
</relocation>
<relocation>
<pattern>io.netty</pattern>
<shadedPattern>shade.io.netty</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>shade.com.google.common</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
<!-- Code coverage test -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.7</version>
<configuration>
<formats>
<format>html</format>
<format>xml</format>
</formats>
<check />
</configuration>
</plugin>
<!-- Google-java-format -->
<plugin>
<groupId>com.coveo</groupId>
<artifactId>fmt-maven-plugin</artifactId>
<version>2.6.0</version>
<configuration>
<sourceDirectory>src/main/java</sourceDirectory>
<testSourceDirectory>src/test/java</testSourceDirectory>
<verbose>true</verbose>
<filesNamePattern>.*\.java</filesNamePattern>
<skip>false</skip>
<skipSortingImports>false</skipSortingImports>
<style>google</style>
</configuration>
<executions>
<execution>
<goals>
<goal>format</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

34
scripts/proto.sh Executable file
View File

@ -0,0 +1,34 @@
#!/bin/sh
#
# Copyright 2017 PingCAP, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.
#
if [ -d "kvproto" ]; then
# cd kvproto; git pull origin master; cd ..
cd kvproto; git checkout 497dd34f807bccdb5bd01009c62c5fd5ced404f3; cd ..
else
git clone https://github.com/pingcap/kvproto; cd kvproto; git checkout 497dd34f807bccdb5bd01009c62c5fd5ced404f3; cd ..
fi
if [ -d "raft-rs" ]; then
cd raft-rs; git pull origin master; cd ..
else
git clone https://github.com/pingcap/raft-rs
fi
if [ -d "tipb" ]; then
cd tipb; git pull origin master; cd ..
else
git clone https://github.com/pingcap/tipb
fi

1
shading_rule Normal file
View File

@ -0,0 +1 @@
rule io.netty.** io.netty.netty4pingcap.@1

152
src/main/java/com/pingcap/tikv/AbstractGRPCClient.java Normal file
View File

@ -0,0 +1,152 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv;
import static io.grpc.stub.ClientCalls.asyncBidiStreamingCall;
import static io.grpc.stub.ClientCalls.blockingServerStreamingCall;
import com.pingcap.tikv.operation.ErrorHandler;
import com.pingcap.tikv.policy.RetryMaxMs.Builder;
import com.pingcap.tikv.policy.RetryPolicy;
import com.pingcap.tikv.streaming.StreamingResponse;
import com.pingcap.tikv.util.BackOffer;
import io.grpc.MethodDescriptor;
import io.grpc.stub.AbstractStub;
import io.grpc.stub.ClientCalls;
import io.grpc.stub.StreamObserver;
import java.util.function.Supplier;
import org.apache.log4j.Logger;
public abstract class AbstractGRPCClient<
BlockingStubT extends AbstractStub<BlockingStubT>, StubT extends AbstractStub<StubT>>
implements AutoCloseable {
protected final Logger logger = Logger.getLogger(this.getClass());
protected TiSession session;
protected TiConfiguration conf;
protected AbstractGRPCClient(TiSession session) {
this.session = session;
this.conf = session.getConf();
}
public TiSession getSession() {
return session;
}
public TiConfiguration getConf() {
return conf;
}
// TODO: Seems a little bit messy for lambda part
public <ReqT, RespT> RespT callWithRetry(
BackOffer backOffer,
MethodDescriptor<ReqT, RespT> method,
Supplier<ReqT> requestFactory,
ErrorHandler<RespT> handler) {
if (logger.isTraceEnabled()) {
logger.trace(String.format("Calling %s...", method.getFullMethodName()));
}
RetryPolicy.Builder<RespT> builder = new Builder<>(backOffer);
RespT resp =
builder
.create(handler)
.callWithRetry(
() -> {
BlockingStubT stub = getBlockingStub();
return ClientCalls.blockingUnaryCall(
stub.getChannel(), method, stub.getCallOptions(), requestFactory.get());
},
method.getFullMethodName());
if (logger.isTraceEnabled()) {
logger.trace(String.format("leaving %s...", method.getFullMethodName()));
}
return resp;
}
protected <ReqT, RespT> void callAsyncWithRetry(
BackOffer backOffer,
MethodDescriptor<ReqT, RespT> method,
Supplier<ReqT> requestFactory,
StreamObserver<RespT> responseObserver,
ErrorHandler<RespT> handler) {
logger.debug(String.format("Calling %s...", method.getFullMethodName()));
RetryPolicy.Builder<RespT> builder = new Builder<>(backOffer);
builder
.create(handler)
.callWithRetry(
() -> {
StubT stub = getAsyncStub();
ClientCalls.asyncUnaryCall(
stub.getChannel().newCall(method, stub.getCallOptions()),
requestFactory.get(),
responseObserver);
return null;
},
method.getFullMethodName());
logger.debug(String.format("leaving %s...", method.getFullMethodName()));
}
<ReqT, RespT> StreamObserver<ReqT> callBidiStreamingWithRetry(
BackOffer backOffer,
MethodDescriptor<ReqT, RespT> method,
StreamObserver<RespT> responseObserver,
ErrorHandler<StreamObserver<ReqT>> handler) {
logger.debug(String.format("Calling %s...", method.getFullMethodName()));
RetryPolicy.Builder<StreamObserver<ReqT>> builder = new Builder<>(backOffer);
StreamObserver<ReqT> observer =
builder
.create(handler)
.callWithRetry(
() -> {
StubT stub = getAsyncStub();
return asyncBidiStreamingCall(
stub.getChannel().newCall(method, stub.getCallOptions()), responseObserver);
},
method.getFullMethodName());
logger.debug(String.format("leaving %s...", method.getFullMethodName()));
return observer;
}
public <ReqT, RespT> StreamingResponse callServerStreamingWithRetry(
BackOffer backOffer,
MethodDescriptor<ReqT, RespT> method,
Supplier<ReqT> requestFactory,
ErrorHandler<StreamingResponse> handler) {
logger.debug(String.format("Calling %s...", method.getFullMethodName()));
RetryPolicy.Builder<StreamingResponse> builder = new Builder<>(backOffer);
StreamingResponse response =
builder
.create(handler)
.callWithRetry(
() -> {
BlockingStubT stub = getBlockingStub();
return new StreamingResponse(
blockingServerStreamingCall(
stub.getChannel(), method, stub.getCallOptions(), requestFactory.get()));
},
method.getFullMethodName());
logger.debug(String.format("leaving %s...", method.getFullMethodName()));
return response;
}
protected abstract BlockingStubT getBlockingStub();
protected abstract StubT getAsyncStub();
}

34
src/main/java/com/pingcap/tikv/BUILD Normal file
View File

@ -0,0 +1,34 @@
package(default_visibility = ["//visibility:public"])
java_library(
name = "tikv-java-client-lib",
srcs = glob(
["**/*.java"],
),
deps = [
"//:java",
"@com_fasterxml_jackson_core_jackson_annotations//jar",
"@com_fasterxml_jackson_core_jackson_core//jar",
"@com_fasterxml_jackson_core_jackson_databind//jar",
"@com_google_code_findbugs_jsr305//jar",
"@com_google_code_gson_gson//jar",
"@com_google_errorprone_error_prone_annotations//jar",
"@com_google_guava_guava//jar",
"@com_google_protobuf_protobuf_java//jar",
"@joda_time//jar",
# the following are defined in rules_protobuf
"@org_pubref_rules_protobuf//java:grpc_compiletime_deps",
"@org_pubref_rules_protobuf//java:netty_runtime_deps",
"@org_slf4j_slf4j_api//jar",
"@org_slf4j_jcl_over_slf4j//jar",
"@org_slf4j_jul_to_slf4j//jar",
"@log4j_log4j//jar",
"@net_sf_trove4j_trove4j//jar",
],
)
filegroup(
name = "srcs",
srcs = ["BUILD"] + glob(["**/*.java"]),
)

5
src/main/java/com/pingcap/tikv/Main.java Normal file
View File

@ -0,0 +1,5 @@
package com.pingcap.tikv;
public class Main {
public static void main(String args[]) throws Exception {}
}

374
src/main/java/com/pingcap/tikv/PDClient.java Normal file
View File

@ -0,0 +1,374 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv;
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.net.HostAndPort;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.protobuf.ByteString;
import com.pingcap.tikv.codec.Codec.BytesCodec;
import com.pingcap.tikv.codec.CodecDataOutput;
import com.pingcap.tikv.exception.GrpcException;
import com.pingcap.tikv.exception.TiClientInternalException;
import com.pingcap.tikv.kvproto.Kvrpcpb.IsolationLevel;
import com.pingcap.tikv.kvproto.Metapb.Store;
import com.pingcap.tikv.kvproto.PDGrpc;
import com.pingcap.tikv.kvproto.PDGrpc.PDBlockingStub;
import com.pingcap.tikv.kvproto.PDGrpc.PDStub;
import com.pingcap.tikv.kvproto.Pdpb.GetMembersRequest;
import com.pingcap.tikv.kvproto.Pdpb.GetMembersResponse;
import com.pingcap.tikv.kvproto.Pdpb.GetRegionByIDRequest;
import com.pingcap.tikv.kvproto.Pdpb.GetRegionRequest;
import com.pingcap.tikv.kvproto.Pdpb.GetRegionResponse;
import com.pingcap.tikv.kvproto.Pdpb.GetStoreRequest;
import com.pingcap.tikv.kvproto.Pdpb.GetStoreResponse;
import com.pingcap.tikv.kvproto.Pdpb.RequestHeader;
import com.pingcap.tikv.kvproto.Pdpb.Timestamp;
import com.pingcap.tikv.kvproto.Pdpb.TsoRequest;
import com.pingcap.tikv.kvproto.Pdpb.TsoResponse;
import com.pingcap.tikv.meta.TiTimestamp;
import com.pingcap.tikv.operation.PDErrorHandler;
import com.pingcap.tikv.region.TiRegion;
import com.pingcap.tikv.util.BackOffer;
import com.pingcap.tikv.util.FutureObserver;
import io.grpc.ManagedChannel;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
public class PDClient extends AbstractGRPCClient<PDBlockingStub, PDStub>
implements ReadOnlyPDClient {
private RequestHeader header;
private TsoRequest tsoReq;
private volatile LeaderWrapper leaderWrapper;
private ScheduledExecutorService service;
private IsolationLevel isolationLevel;
private List<HostAndPort> pdAddrs;
@Override
public TiTimestamp getTimestamp(BackOffer backOffer) {
Supplier<TsoRequest> request = () -> tsoReq;
PDErrorHandler<TsoResponse> handler =
new PDErrorHandler<>(r -> r.getHeader().hasError() ? r.getHeader().getError() : null, this);
TsoResponse resp = callWithRetry(backOffer, PDGrpc.METHOD_TSO, request, handler);
Timestamp timestamp = resp.getTimestamp();
return new TiTimestamp(timestamp.getPhysical(), timestamp.getLogical());
}
@Override
public TiRegion getRegionByKey(BackOffer backOffer, ByteString key) {
CodecDataOutput cdo = new CodecDataOutput();
BytesCodec.writeBytes(cdo, key.toByteArray());
ByteString encodedKey = cdo.toByteString();
Supplier<GetRegionRequest> request =
() -> GetRegionRequest.newBuilder().setHeader(header).setRegionKey(encodedKey).build();
PDErrorHandler<GetRegionResponse> handler =
new PDErrorHandler<>(r -> r.getHeader().hasError() ? r.getHeader().getError() : null, this);
GetRegionResponse resp = callWithRetry(backOffer, PDGrpc.METHOD_GET_REGION, request, handler);
return new TiRegion(
resp.getRegion(), resp.getLeader(), conf.getIsolationLevel(), conf.getCommandPriority());
}
@Override
public Future<TiRegion> getRegionByKeyAsync(BackOffer backOffer, ByteString key) {
FutureObserver<TiRegion, GetRegionResponse> responseObserver =
new FutureObserver<>(
resp ->
new TiRegion(
resp.getRegion(),
resp.getLeader(),
conf.getIsolationLevel(),
conf.getCommandPriority()));
Supplier<GetRegionRequest> request =
() -> GetRegionRequest.newBuilder().setHeader(header).setRegionKey(key).build();
PDErrorHandler<GetRegionResponse> handler =
new PDErrorHandler<>(r -> r.getHeader().hasError() ? r.getHeader().getError() : null, this);
callAsyncWithRetry(backOffer, PDGrpc.METHOD_GET_REGION, request, responseObserver, handler);
return responseObserver.getFuture();
}
@Override
public TiRegion getRegionByID(BackOffer backOffer, long id) {
Supplier<GetRegionByIDRequest> request =
() -> GetRegionByIDRequest.newBuilder().setHeader(header).setRegionId(id).build();
PDErrorHandler<GetRegionResponse> handler =
new PDErrorHandler<>(r -> r.getHeader().hasError() ? r.getHeader().getError() : null, this);
GetRegionResponse resp =
callWithRetry(backOffer, PDGrpc.METHOD_GET_REGION_BY_ID, request, handler);
// Instead of using default leader instance, explicitly set no leader to null
return new TiRegion(
resp.getRegion(), resp.getLeader(), conf.getIsolationLevel(), conf.getCommandPriority());
}
@Override
public Future<TiRegion> getRegionByIDAsync(BackOffer backOffer, long id) {
FutureObserver<TiRegion, GetRegionResponse> responseObserver =
new FutureObserver<>(
resp ->
new TiRegion(
resp.getRegion(),
resp.getLeader(),
conf.getIsolationLevel(),
conf.getCommandPriority()));
Supplier<GetRegionByIDRequest> request =
() -> GetRegionByIDRequest.newBuilder().setHeader(header).setRegionId(id).build();
PDErrorHandler<GetRegionResponse> handler =
new PDErrorHandler<>(r -> r.getHeader().hasError() ? r.getHeader().getError() : null, this);
callAsyncWithRetry(
backOffer, PDGrpc.METHOD_GET_REGION_BY_ID, request, responseObserver, handler);
return responseObserver.getFuture();
}
@Override
public Store getStore(BackOffer backOffer, long storeId) {
Supplier<GetStoreRequest> request =
() -> GetStoreRequest.newBuilder().setHeader(header).setStoreId(storeId).build();
PDErrorHandler<GetStoreResponse> handler =
new PDErrorHandler<>(r -> r.getHeader().hasError() ? r.getHeader().getError() : null, this);
GetStoreResponse resp = callWithRetry(backOffer, PDGrpc.METHOD_GET_STORE, request, handler);
return resp.getStore();
}
@Override
public Future<Store> getStoreAsync(BackOffer backOffer, long storeId) {
FutureObserver<Store, GetStoreResponse> responseObserver =
new FutureObserver<>(GetStoreResponse::getStore);
Supplier<GetStoreRequest> request =
() -> GetStoreRequest.newBuilder().setHeader(header).setStoreId(storeId).build();
PDErrorHandler<GetStoreResponse> handler =
new PDErrorHandler<>(r -> r.getHeader().hasError() ? r.getHeader().getError() : null, this);
callAsyncWithRetry(backOffer, PDGrpc.METHOD_GET_STORE, request, responseObserver, handler);
return responseObserver.getFuture();
}
@Override
public void close() throws InterruptedException {
if (service != null) {
service.shutdownNow();
}
if (getLeaderWrapper() != null) {
getLeaderWrapper().close();
}
}
public static ReadOnlyPDClient create(TiSession session) {
return createRaw(session);
}
@VisibleForTesting
RequestHeader getHeader() {
return header;
}
@VisibleForTesting
LeaderWrapper getLeaderWrapper() {
return leaderWrapper;
}
class LeaderWrapper {
private final String leaderInfo;
private final PDBlockingStub blockingStub;
private final PDStub asyncStub;
private final long createTime;
LeaderWrapper(
String leaderInfo,
PDGrpc.PDBlockingStub blockingStub,
PDGrpc.PDStub asyncStub,
long createTime) {
this.leaderInfo = leaderInfo;
this.blockingStub = blockingStub;
this.asyncStub = asyncStub;
this.createTime = createTime;
}
String getLeaderInfo() {
return leaderInfo;
}
PDBlockingStub getBlockingStub() {
return blockingStub;
}
PDStub getAsyncStub() {
return asyncStub;
}
long getCreateTime() {
return createTime;
}
void close() {}
}
public GetMembersResponse getMembers(HostAndPort url) {
try {
ManagedChannel probChan = session.getChannel(url.getHostText() + ":" + url.getPort());
PDGrpc.PDBlockingStub stub = PDGrpc.newBlockingStub(probChan);
GetMembersRequest request =
GetMembersRequest.newBuilder().setHeader(RequestHeader.getDefaultInstance()).build();
return stub.getMembers(request);
} catch (Exception e) {
logger.warn("failed to get member from pd server.", e);
}
return null;
}
private synchronized boolean switchLeader(List<String> leaderURLs) {
if (leaderURLs.isEmpty()) return false;
String leaderUrlStr = leaderURLs.get(0);
// TODO: Why not strip protocol info on server side since grpc does not need it
if (leaderWrapper != null && leaderUrlStr.equals(leaderWrapper.getLeaderInfo())) {
return true;
}
// switch leader
return createLeaderWrapper(leaderUrlStr);
}
private boolean createLeaderWrapper(String leaderUrlStr) {
try {
URL tURL = new URL(leaderUrlStr);
HostAndPort newLeader = HostAndPort.fromParts(tURL.getHost(), tURL.getPort());
leaderUrlStr = newLeader.toString();
if (leaderWrapper != null && leaderUrlStr.equals(leaderWrapper.getLeaderInfo())) {
return true;
}
// create new Leader
ManagedChannel clientChannel = session.getChannel(leaderUrlStr);
leaderWrapper =
new LeaderWrapper(
leaderUrlStr,
PDGrpc.newBlockingStub(clientChannel),
PDGrpc.newStub(clientChannel),
System.nanoTime());
} catch (MalformedURLException e) {
logger.error("Error updating leader.", e);
return false;
}
logger.info(String.format("Switched to new leader: %s", leaderWrapper));
return true;
}
public void updateLeader() {
for (HostAndPort url : this.pdAddrs) {
// since resp is null, we need update leader's address by walking through all pd server.
GetMembersResponse resp = getMembers(url);
if (resp == null) {
continue;
}
// if leader is switched, just return.
if (switchLeader(resp.getLeader().getClientUrlsList())) {
return;
}
}
throw new TiClientInternalException(
"already tried all address on file, but not leader found yet.");
}
@Override
protected PDBlockingStub getBlockingStub() {
if (leaderWrapper == null) {
throw new GrpcException("PDClient may not be initialized");
}
return leaderWrapper
.getBlockingStub()
.withDeadlineAfter(getConf().getTimeout(), getConf().getTimeoutUnit());
}
@Override
protected PDStub getAsyncStub() {
if (leaderWrapper == null) {
throw new GrpcException("PDClient may not be initialized");
}
return leaderWrapper
.getAsyncStub()
.withDeadlineAfter(getConf().getTimeout(), getConf().getTimeoutUnit());
}
private PDClient(TiSession session) {
super(session);
}
private void initCluster() {
GetMembersResponse resp = null;
List<HostAndPort> pdAddrs = getSession().getConf().getPdAddrs();
for (HostAndPort u : pdAddrs) {
resp = getMembers(u);
if (resp != null) {
break;
}
}
checkNotNull(resp, "Failed to init client for PD cluster.");
long clusterId = resp.getHeader().getClusterId();
header = RequestHeader.newBuilder().setClusterId(clusterId).build();
tsoReq = TsoRequest.newBuilder().setHeader(header).setCount(1).build();
this.pdAddrs = pdAddrs;
createLeaderWrapper(resp.getLeader().getClientUrls(0));
service =
Executors.newSingleThreadScheduledExecutor(
new ThreadFactoryBuilder().setDaemon(true).build());
service.scheduleAtFixedRate(
() -> {
// Wrap this with a try catch block in case schedule update fails
try {
updateLeader();
} catch (Exception e) {
logger.warn("Update leader failed", e);
}
},
1,
1,
TimeUnit.MINUTES);
}
static PDClient createRaw(TiSession session) {
PDClient client = null;
try {
client = new PDClient(session);
client.initCluster();
} catch (Exception e) {
if (client != null) {
try {
client.close();
} catch (InterruptedException ignore) {
}
}
throw e;
}
return client;
}
}

120
src/main/java/com/pingcap/tikv/RawKVClient.java Normal file
View File

@ -0,0 +1,120 @@
package com.pingcap.tikv;
import com.google.protobuf.ByteString;
import com.pingcap.tikv.kvproto.Kvrpcpb;
import com.pingcap.tikv.kvproto.Metapb;
import com.pingcap.tikv.operation.iterator.RawScanIterator;
import com.pingcap.tikv.region.RegionManager;
import com.pingcap.tikv.region.RegionStoreClient;
import com.pingcap.tikv.region.TiRegion;
import com.pingcap.tikv.util.BackOffer;
import com.pingcap.tikv.util.ConcreteBackOffer;
import com.pingcap.tikv.util.Pair;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
public class RawKVClient {
private static final String DEFAULT_PD_ADDRESS = "127.0.0.1:2379";
private final TiSession session;
private final RegionManager regionManager;
private RawKVClient(String addresses) {
session = TiSession.create(TiConfiguration.createDefault(addresses));
regionManager = session.getRegionManager();
}
private RawKVClient() {
this(DEFAULT_PD_ADDRESS);
}
public static RawKVClient create() {
return new RawKVClient();
}
public static RawKVClient create(String address) {
return new RawKVClient(address);
}
/**
* Put a raw key-value pair to TiKV
*
* @param key raw key
* @param value raw value
*/
public void put(ByteString key, ByteString value) {
Pair<TiRegion, Metapb.Store> pair = regionManager.getRegionStorePairByRawKey(key);
RegionStoreClient client = RegionStoreClient.create(pair.first, pair.second, session);
client.rawPut(defaultBackOff(), key, value);
}
/**
* Get a raw key-value pair from TiKV if key exists
*
* @param key raw key
* @return a ByteString value if key exists, ByteString.EMPTY if key does not exist
*/
public ByteString get(ByteString key) {
Pair<TiRegion, Metapb.Store> pair = regionManager.getRegionStorePairByRawKey(key);
RegionStoreClient client = RegionStoreClient.create(pair.first, pair.second, session);
return client.rawGet(defaultBackOff(), key);
}
/**
* Scan raw key-value pairs from TiKV in range [startKey, endKey)
*
* @param startKey raw start key, inclusive
* @param endKey raw end key, exclusive
* @return list of key-value pairs in range
*/
public List<Kvrpcpb.KvPair> scan(ByteString startKey, ByteString endKey) {
Iterator<Kvrpcpb.KvPair> iterator = rawScanIterator(startKey, endKey);
List<Kvrpcpb.KvPair> result = new ArrayList<>();
iterator.forEachRemaining(result::add);
return result;
}
/**
* Scan raw key-value pairs from TiKV in range [startKey, endKey)
*
* @param startKey raw start key, inclusive
* @param limit limit of key-value pairs
* @return list of key-value pairs in range
*/
public List<Kvrpcpb.KvPair> scan(ByteString startKey, int limit) {
Iterator<Kvrpcpb.KvPair> iterator = rawScanIterator(startKey, limit);
List<Kvrpcpb.KvPair> result = new ArrayList<>();
iterator.forEachRemaining(result::add);
return result;
}
/**
* Delete a raw key-value pair from TiKV if key exists
*
* @param key raw key to be deleted
*/
public void delete(ByteString key) {
TiRegion region = regionManager.getRegionByRawKey(key);
Kvrpcpb.Context context =
Kvrpcpb.Context.newBuilder()
.setRegionId(region.getId())
.setRegionEpoch(region.getRegionEpoch())
.setPeer(region.getLeader())
.build();
Pair<TiRegion, Metapb.Store> pair = regionManager.getRegionStorePairByRawKey(key);
RegionStoreClient client = RegionStoreClient.create(pair.first, pair.second, session);
client.rawDelete(defaultBackOff(), key, context);
}
private Iterator<Kvrpcpb.KvPair> rawScanIterator(ByteString startKey, ByteString endKey) {
return new RawScanIterator(startKey, endKey, Integer.MAX_VALUE, session);
}
private Iterator<Kvrpcpb.KvPair> rawScanIterator(ByteString startKey, int limit) {
return new RawScanIterator(startKey, ByteString.EMPTY, limit, session);
}
private BackOffer defaultBackOff() {
return ConcreteBackOffer.newCustomBackOff(1000);
}
}

69
src/main/java/com/pingcap/tikv/ReadOnlyPDClient.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv;
import com.google.protobuf.ByteString;
import com.pingcap.tikv.kvproto.Metapb.Store;
import com.pingcap.tikv.meta.TiTimestamp;
import com.pingcap.tikv.region.TiRegion;
import com.pingcap.tikv.util.BackOffer;
import java.util.concurrent.Future;
/** Readonly PD client including only reading related interface Supposed for TiDB-like use cases */
public interface ReadOnlyPDClient {
/**
* Get Timestamp from Placement Driver
*
* @return a timestamp object
*/
TiTimestamp getTimestamp(BackOffer backOffer);
/**
* Get Region from PD by key specified
*
* @param key key in bytes for locating a region
* @return the region whose startKey and endKey range covers the given key
*/
TiRegion getRegionByKey(BackOffer backOffer, ByteString key);
Future<TiRegion> getRegionByKeyAsync(BackOffer backOffer, ByteString key);
/**
* Get Region by Region Id
*
* @param id Region Id
* @return the region corresponding to the given Id
*/
TiRegion getRegionByID(BackOffer backOffer, long id);
Future<TiRegion> getRegionByIDAsync(BackOffer backOffer, long id);
/**
* Get Store by StoreId
*
* @param storeId StoreId
* @return the Store corresponding to the given Id
*/
Store getStore(BackOffer backOffer, long storeId);
Future<Store> getStoreAsync(BackOffer backOffer, long storeId);
/** Close underlining resources */
void close() throws InterruptedException;
/** Get associated session * @return the session associated to client */
TiSession getSession();
}

168
src/main/java/com/pingcap/tikv/Snapshot.java Normal file
View File

@ -0,0 +1,168 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv;
import static com.pingcap.tikv.operation.iterator.CoprocessIterator.getHandleIterator;
import static com.pingcap.tikv.operation.iterator.CoprocessIterator.getRowIterator;
import static com.pingcap.tikv.util.KeyRangeUtils.makeRange;
import com.google.common.collect.Range;
import com.google.protobuf.ByteString;
import com.pingcap.tikv.exception.TiClientInternalException;
import com.pingcap.tikv.key.Key;
import com.pingcap.tikv.kvproto.Kvrpcpb.KvPair;
import com.pingcap.tikv.kvproto.Metapb.Store;
import com.pingcap.tikv.meta.TiDAGRequest;
import com.pingcap.tikv.meta.TiTimestamp;
import com.pingcap.tikv.operation.iterator.ConcreteScanIterator;
import com.pingcap.tikv.operation.iterator.IndexScanIterator;
import com.pingcap.tikv.region.RegionStoreClient;
import com.pingcap.tikv.region.TiRegion;
import com.pingcap.tikv.row.Row;
import com.pingcap.tikv.util.BackOffer;
import com.pingcap.tikv.util.ConcreteBackOffer;
import com.pingcap.tikv.util.Pair;
import com.pingcap.tikv.util.RangeSplitter;
import com.pingcap.tikv.util.RangeSplitter.RegionTask;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
public class Snapshot {
private final TiTimestamp timestamp;
private final TiSession session;
private final TiConfiguration conf;
public Snapshot(TiTimestamp timestamp, TiSession session) {
this.timestamp = timestamp;
this.session = session;
this.conf = session.getConf();
}
public TiSession getSession() {
return session;
}
public long getVersion() {
return timestamp.getVersion();
}
public TiTimestamp getTimestamp() {
return timestamp;
}
public byte[] get(byte[] key) {
ByteString keyString = ByteString.copyFrom(key);
ByteString value = get(keyString);
return value.toByteArray();
}
public ByteString get(ByteString key) {
Pair<TiRegion, Store> pair = session.getRegionManager().getRegionStorePairByKey(key);
RegionStoreClient client = RegionStoreClient.create(pair.first, pair.second, getSession());
// TODO: Need to deal with lock error after grpc stable
return client.get(ConcreteBackOffer.newGetBackOff(), key, timestamp.getVersion());
}
/**
* Issue a table read request
*
* @param dagRequest DAG request for coprocessor
* @return a Iterator that contains all result from this select request.
*/
public Iterator<Row> tableRead(TiDAGRequest dagRequest) {
if (dagRequest.isIndexScan()) {
Iterator<Long> iter =
getHandleIterator(
dagRequest,
RangeSplitter.newSplitter(session.getRegionManager())
.splitRangeByRegion(dagRequest.getRanges()),
session);
return new IndexScanIterator(this, dagRequest, iter);
} else {
return getRowIterator(
dagRequest,
RangeSplitter.newSplitter(session.getRegionManager())
.splitRangeByRegion(dagRequest.getRanges()),
session);
}
}
/**
* Below is lower level API for env like Spark which already did key range split Perform table
* scan
*
* @param dagRequest DAGRequest for coprocessor
* @param task RegionTask of the coprocessor request to send
* @return Row iterator to iterate over resulting rows
*/
public Iterator<Row> tableRead(TiDAGRequest dagRequest, List<RegionTask> task) {
if (dagRequest.isDoubleRead()) {
Iterator<Long> iter = getHandleIterator(dagRequest, task, session);
return new IndexScanIterator(this, dagRequest, iter);
} else {
return getRowIterator(dagRequest, task, session);
}
}
/**
* Below is lower level API for env like Spark which already did key range split Perform handle
* scan
*
* @param dagRequest DAGRequest for coprocessor
* @param tasks RegionTask of the coprocessor request to send
* @return Row iterator to iterate over resulting rows
*/
public Iterator<Long> indexHandleRead(TiDAGRequest dagRequest, List<RegionTask> tasks) {
return getHandleIterator(dagRequest, tasks, session);
}
public Iterator<KvPair> scan(ByteString startKey) {
return new ConcreteScanIterator(startKey, session, timestamp.getVersion());
}
// TODO: Need faster implementation, say concurrent version
// Assume keys sorted
public List<KvPair> batchGet(List<ByteString> keys) {
TiRegion curRegion = null;
Range<Key> curKeyRange = null;
Pair<TiRegion, Store> lastPair;
List<ByteString> keyBuffer = new ArrayList<>();
List<KvPair> result = new ArrayList<>(keys.size());
BackOffer backOffer = ConcreteBackOffer.newBatchGetMaxBackOff();
for (ByteString key : keys) {
if (curRegion == null || !curKeyRange.contains(Key.toRawKey(key))) {
Pair<TiRegion, Store> pair = session.getRegionManager().getRegionStorePairByKey(key);
lastPair = pair;
curRegion = pair.first;
curKeyRange = makeRange(curRegion.getStartKey(), curRegion.getEndKey());
try (RegionStoreClient client =
RegionStoreClient.create(lastPair.first, lastPair.second, getSession())) {
List<KvPair> partialResult =
client.batchGet(backOffer, keyBuffer, timestamp.getVersion());
// TODO: Add lock check
result.addAll(partialResult);
} catch (Exception e) {
throw new TiClientInternalException("Error Closing Store client.", e);
}
keyBuffer = new ArrayList<>();
keyBuffer.add(key);
}
}
return result;
}
}

219
src/main/java/com/pingcap/tikv/TiConfiguration.java Normal file
View File

@ -0,0 +1,219 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv;
import com.google.common.collect.ImmutableList;
import com.google.common.net.HostAndPort;
import com.pingcap.tikv.kvproto.Kvrpcpb.CommandPri;
import com.pingcap.tikv.kvproto.Kvrpcpb.IsolationLevel;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
public class TiConfiguration implements Serializable {
private static final int DEF_TIMEOUT = 10;
private static final TimeUnit DEF_TIMEOUT_UNIT = TimeUnit.MINUTES;
private static final int DEF_SCAN_BATCH_SIZE = 100;
private static final boolean DEF_IGNORE_TRUNCATE = true;
private static final boolean DEF_TRUNCATE_AS_WARNING = false;
private static final int DEF_META_RELOAD_PERIOD = 10;
private static final TimeUnit DEF_META_RELOAD_UNIT = TimeUnit.SECONDS;
private static final int DEF_MAX_FRAME_SIZE = 268435456 * 2; // 256 * 2 MB
private static final int DEF_INDEX_SCAN_BATCH_SIZE = 20000;
// if keyRange size per request exceeds this limit, the request might be too large to be accepted
// by TiKV(maximum request size accepted by TiKV is around 1MB)
private static final int MAX_REQUEST_KEY_RANGE_SIZE = 20000;
private static final int DEF_INDEX_SCAN_CONCURRENCY = 5;
private static final int DEF_TABLE_SCAN_CONCURRENCY = 512;
private static final CommandPri DEF_COMMAND_PRIORITY = CommandPri.Low;
private static final IsolationLevel DEF_ISOLATION_LEVEL = IsolationLevel.RC;
private static final boolean DEF_SHOW_ROWID = false;
private static final String DEF_DB_PREFIX = "";
private int timeout = DEF_TIMEOUT;
private TimeUnit timeoutUnit = DEF_TIMEOUT_UNIT;
private boolean ignoreTruncate = DEF_IGNORE_TRUNCATE;
private boolean truncateAsWarning = DEF_TRUNCATE_AS_WARNING;
private TimeUnit metaReloadUnit = DEF_META_RELOAD_UNIT;
private int metaReloadPeriod = DEF_META_RELOAD_PERIOD;
private int maxFrameSize = DEF_MAX_FRAME_SIZE;
private List<HostAndPort> pdAddrs = new ArrayList<>();
private int indexScanBatchSize = DEF_INDEX_SCAN_BATCH_SIZE;
private int indexScanConcurrency = DEF_INDEX_SCAN_CONCURRENCY;
private int tableScanConcurrency = DEF_TABLE_SCAN_CONCURRENCY;
private CommandPri commandPriority = DEF_COMMAND_PRIORITY;
private IsolationLevel isolationLevel = DEF_ISOLATION_LEVEL;
private int maxRequestKeyRangeSize = MAX_REQUEST_KEY_RANGE_SIZE;
private boolean showRowId = DEF_SHOW_ROWID;
private String dbPrefix = DEF_DB_PREFIX;
public static TiConfiguration createDefault(String pdAddrsStr) {
Objects.requireNonNull(pdAddrsStr, "pdAddrsStr is null");
TiConfiguration conf = new TiConfiguration();
conf.pdAddrs = strToHostAndPort(pdAddrsStr);
return conf;
}
private static List<HostAndPort> strToHostAndPort(String addressStr) {
Objects.requireNonNull(addressStr);
String[] addrs = addressStr.split(",");
ImmutableList.Builder<HostAndPort> addrsBuilder = ImmutableList.builder();
for (String addr : addrs) {
addrsBuilder.add(HostAndPort.fromString(addr));
}
return addrsBuilder.build();
}
public int getTimeout() {
return timeout;
}
public TiConfiguration setTimeout(int timeout) {
this.timeout = timeout;
return this;
}
public TimeUnit getTimeoutUnit() {
return timeoutUnit;
}
public TimeUnit getMetaReloadPeriodUnit() {
return metaReloadUnit;
}
public TiConfiguration setMetaReloadPeriodUnit(TimeUnit timeUnit) {
this.metaReloadUnit = timeUnit;
return this;
}
public TiConfiguration setMetaReloadPeriod(int metaReloadPeriod) {
this.metaReloadPeriod = metaReloadPeriod;
return this;
}
public int getMetaReloadPeriod() {
return metaReloadPeriod;
}
public TiConfiguration setTimeoutUnit(TimeUnit timeoutUnit) {
this.timeoutUnit = timeoutUnit;
return this;
}
public List<HostAndPort> getPdAddrs() {
return pdAddrs;
}
public int getScanBatchSize() {
return DEF_SCAN_BATCH_SIZE;
}
boolean isIgnoreTruncate() {
return ignoreTruncate;
}
public TiConfiguration setIgnoreTruncate(boolean ignoreTruncate) {
this.ignoreTruncate = ignoreTruncate;
return this;
}
boolean isTruncateAsWarning() {
return truncateAsWarning;
}
public TiConfiguration setTruncateAsWarning(boolean truncateAsWarning) {
this.truncateAsWarning = truncateAsWarning;
return this;
}
public int getMaxFrameSize() {
return maxFrameSize;
}
public TiConfiguration setMaxFrameSize(int maxFrameSize) {
this.maxFrameSize = maxFrameSize;
return this;
}
public int getIndexScanBatchSize() {
return indexScanBatchSize;
}
public void setIndexScanBatchSize(int indexScanBatchSize) {
this.indexScanBatchSize = indexScanBatchSize;
}
public int getIndexScanConcurrency() {
return indexScanConcurrency;
}
public void setIndexScanConcurrency(int indexScanConcurrency) {
this.indexScanConcurrency = indexScanConcurrency;
}
public int getTableScanConcurrency() {
return tableScanConcurrency;
}
public void setTableScanConcurrency(int tableScanConcurrency) {
this.tableScanConcurrency = tableScanConcurrency;
}
public CommandPri getCommandPriority() {
return commandPriority;
}
public void setCommandPriority(CommandPri commandPriority) {
this.commandPriority = commandPriority;
}
public IsolationLevel getIsolationLevel() {
return isolationLevel;
}
public void setIsolationLevel(IsolationLevel isolationLevel) {
this.isolationLevel = isolationLevel;
}
public int getMaxRequestKeyRangeSize() {
return maxRequestKeyRangeSize;
}
public void setMaxRequestKeyRangeSize(int maxRequestKeyRangeSize) {
if (maxRequestKeyRangeSize <= 0) {
throw new IllegalArgumentException("Key range size cannot be less than 1");
}
this.maxRequestKeyRangeSize = maxRequestKeyRangeSize;
}
public void setShowRowId(boolean flag) {
this.showRowId = flag;
}
public boolean ifShowRowId() {
return showRowId;
}
public String getDBPrefix() {
return dbPrefix;
}
public void setDBPrefix(String dbPrefix) {
this.dbPrefix = dbPrefix;
}
}

187
src/main/java/com/pingcap/tikv/TiSession.java Normal file
View File

@ -0,0 +1,187 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv;
import com.google.common.net.HostAndPort;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.pingcap.tikv.catalog.Catalog;
import com.pingcap.tikv.event.CacheInvalidateEvent;
import com.pingcap.tikv.meta.TiTimestamp;
import com.pingcap.tikv.region.RegionManager;
import com.pingcap.tikv.util.ConcreteBackOffer;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
public class TiSession implements AutoCloseable {
private static final Map<String, ManagedChannel> connPool = new HashMap<>();
private final TiConfiguration conf;
private Function<CacheInvalidateEvent, Void> cacheInvalidateCallback;
// below object creation is either heavy or making connection (pd), pending for lazy loading
private volatile RegionManager regionManager;
private volatile PDClient client;
private volatile Catalog catalog;
private volatile ExecutorService indexScanThreadPool;
private volatile ExecutorService tableScanThreadPool;
public TiSession(TiConfiguration conf) {
this.conf = conf;
}
public TiConfiguration getConf() {
return conf;
}
public TiTimestamp getTimestamp() {
return getPDClient().getTimestamp(ConcreteBackOffer.newTsoBackOff());
}
public Snapshot createSnapshot() {
return new Snapshot(getTimestamp(), this);
}
public Snapshot createSnapshot(TiTimestamp ts) {
return new Snapshot(ts, this);
}
public PDClient getPDClient() {
PDClient res = client;
if (res == null) {
synchronized (this) {
if (client == null) {
client = PDClient.createRaw(this);
}
res = client;
}
}
return res;
}
public Catalog getCatalog() {
Catalog res = catalog;
if (res == null) {
synchronized (this) {
if (catalog == null) {
catalog =
new Catalog(
this::createSnapshot,
conf.getMetaReloadPeriod(),
conf.getMetaReloadPeriodUnit(),
conf.ifShowRowId(),
conf.getDBPrefix());
}
res = catalog;
}
}
return res;
}
public synchronized RegionManager getRegionManager() {
RegionManager res = regionManager;
if (res == null) {
synchronized (this) {
if (regionManager == null) {
regionManager = new RegionManager(getPDClient());
}
res = regionManager;
}
}
return res;
}
public synchronized ManagedChannel getChannel(String addressStr) {
ManagedChannel channel = connPool.get(addressStr);
if (channel == null) {
HostAndPort address;
try {
address = HostAndPort.fromString(addressStr);
} catch (Exception e) {
throw new IllegalArgumentException("failed to form address");
}
// Channel should be lazy without actual connection until first call
// So a coarse grain lock is ok here
channel =
ManagedChannelBuilder.forAddress(address.getHostText(), address.getPort())
.maxInboundMessageSize(conf.getMaxFrameSize())
.usePlaintext(true)
.idleTimeout(60, TimeUnit.SECONDS)
.build();
connPool.put(addressStr, channel);
}
return channel;
}
public ExecutorService getThreadPoolForIndexScan() {
ExecutorService res = indexScanThreadPool;
if (res == null) {
synchronized (this) {
if (indexScanThreadPool == null) {
indexScanThreadPool =
Executors.newFixedThreadPool(
conf.getIndexScanConcurrency(),
new ThreadFactoryBuilder().setDaemon(true).build());
}
res = indexScanThreadPool;
}
}
return res;
}
public ExecutorService getThreadPoolForTableScan() {
ExecutorService res = tableScanThreadPool;
if (res == null) {
synchronized (this) {
if (tableScanThreadPool == null) {
tableScanThreadPool =
Executors.newFixedThreadPool(
conf.getTableScanConcurrency(),
new ThreadFactoryBuilder().setDaemon(true).build());
}
res = tableScanThreadPool;
}
}
return res;
}
public static TiSession create(TiConfiguration conf) {
return new TiSession(conf);
}
public Function<CacheInvalidateEvent, Void> getCacheInvalidateCallback() {
return cacheInvalidateCallback;
}
/**
* This is used for setting call back function to invalidate cache information
*
* @param callBackFunc callback function
*/
public void injectCallBackFunc(Function<CacheInvalidateEvent, Void> callBackFunc) {
this.cacheInvalidateCallback = callBackFunc;
}
@Override
public void close() throws Exception {
getThreadPoolForTableScan().shutdownNow();
getThreadPoolForIndexScan().shutdownNow();
}
}

226
src/main/java/com/pingcap/tikv/catalog/Catalog.java Normal file
View File

@ -0,0 +1,226 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.catalog;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.pingcap.tikv.Snapshot;
import com.pingcap.tikv.meta.TiDBInfo;
import com.pingcap.tikv.meta.TiTableInfo;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.apache.log4j.Logger;
public class Catalog implements AutoCloseable {
private Supplier<Snapshot> snapshotProvider;
private ScheduledExecutorService service;
private CatalogCache metaCache;
private final boolean showRowId;
private final String dbPrefix;
private final Logger logger = Logger.getLogger(this.getClass());
@Override
public void close() throws Exception {
if (service != null) {
service.shutdown();
}
}
private static class CatalogCache {
private CatalogCache(CatalogTransaction transaction, String dbPrefix, boolean loadTables) {
this.transaction = transaction;
this.dbPrefix = dbPrefix;
this.dbCache = loadDatabases(loadTables);
this.tableCache = new ConcurrentHashMap<>();
this.currentVersion = transaction.getLatestSchemaVersion();
}
private final Map<String, TiDBInfo> dbCache;
private final ConcurrentHashMap<TiDBInfo, Map<String, TiTableInfo>> tableCache;
private CatalogTransaction transaction;
private long currentVersion;
private final String dbPrefix;
public CatalogTransaction getTransaction() {
return transaction;
}
public long getVersion() {
return currentVersion;
}
public TiDBInfo getDatabase(String name) {
Objects.requireNonNull(name, "name is null");
return dbCache.get(name.toLowerCase());
}
public List<TiDBInfo> listDatabases() {
return ImmutableList.copyOf(dbCache.values());
}
public List<TiTableInfo> listTables(TiDBInfo db) {
Map<String, TiTableInfo> tableMap = tableCache.get(db);
if (tableMap == null) {
tableMap = loadTables(db);
}
return ImmutableList.copyOf(tableMap.values());
}
public TiTableInfo getTable(TiDBInfo db, String tableName) {
Map<String, TiTableInfo> tableMap = tableCache.get(db);
if (tableMap == null) {
tableMap = loadTables(db);
}
return tableMap.get(tableName.toLowerCase());
}
private Map<String, TiTableInfo> loadTables(TiDBInfo db) {
List<TiTableInfo> tables = transaction.getTables(db.getId());
ImmutableMap.Builder<String, TiTableInfo> builder = ImmutableMap.builder();
for (TiTableInfo table : tables) {
builder.put(table.getName().toLowerCase(), table);
}
Map<String, TiTableInfo> tableMap = builder.build();
tableCache.put(db, tableMap);
return tableMap;
}
private Map<String, TiDBInfo> loadDatabases(boolean loadTables) {
HashMap<String, TiDBInfo> newDBCache = new HashMap<>();
List<TiDBInfo> databases = transaction.getDatabases();
databases.forEach(
db -> {
TiDBInfo newDBInfo = db.rename(dbPrefix + db.getName());
newDBCache.put(newDBInfo.getName().toLowerCase(), newDBInfo);
if (loadTables) {
loadTables(newDBInfo);
}
});
return newDBCache;
}
}
public Catalog(
Supplier<Snapshot> snapshotProvider,
int refreshPeriod,
TimeUnit periodUnit,
boolean showRowId,
String dbPrefix) {
this.snapshotProvider = Objects.requireNonNull(snapshotProvider, "Snapshot Provider is null");
this.showRowId = showRowId;
this.dbPrefix = dbPrefix;
metaCache = new CatalogCache(new CatalogTransaction(snapshotProvider.get()), dbPrefix, false);
service =
Executors.newSingleThreadScheduledExecutor(
new ThreadFactoryBuilder().setDaemon(true).build());
service.scheduleAtFixedRate(
() -> {
// Wrap this with a try catch block in case schedule update fails
try {
reloadCache();
} catch (Exception e) {
logger.warn("Reload Cache failed", e);
}
},
refreshPeriod,
refreshPeriod,
periodUnit);
}
public void reloadCache(boolean loadTables) {
Snapshot snapshot = snapshotProvider.get();
CatalogTransaction newTrx = new CatalogTransaction(snapshot);
long latestVersion = newTrx.getLatestSchemaVersion();
if (latestVersion > metaCache.getVersion()) {
metaCache = new CatalogCache(newTrx, dbPrefix, loadTables);
}
}
@VisibleForTesting
public void reloadCache() {
reloadCache(false);
}
public List<TiDBInfo> listDatabases() {
return metaCache.listDatabases();
}
public List<TiTableInfo> listTables(TiDBInfo database) {
Objects.requireNonNull(database, "database is null");
if (showRowId) {
return metaCache
.listTables(database)
.stream()
.map(TiTableInfo::copyTableWithRowId)
.collect(Collectors.toList());
} else {
return metaCache.listTables(database);
}
}
public TiDBInfo getDatabase(String dbName) {
Objects.requireNonNull(dbName, "dbName is null");
return metaCache.getDatabase(dbName);
}
public TiTableInfo getTable(String dbName, String tableName) {
TiDBInfo database = getDatabase(dbName);
if (database == null) {
return null;
}
return getTable(database, tableName);
}
public TiTableInfo getTable(TiDBInfo database, String tableName) {
Objects.requireNonNull(database, "database is null");
Objects.requireNonNull(tableName, "tableName is null");
TiTableInfo table = metaCache.getTable(database, tableName);
if (showRowId) {
return table.copyTableWithRowId();
} else {
return table;
}
}
@VisibleForTesting
public TiTableInfo getTable(TiDBInfo database, long tableId) {
Objects.requireNonNull(database, "database is null");
Collection<TiTableInfo> tables = listTables(database);
for (TiTableInfo table : tables) {
if (table.getId() == tableId) {
if (showRowId) {
return table.copyTableWithRowId();
} else {
return table;
}
}
}
return null;
}
}

183
src/main/java/com/pingcap/tikv/catalog/CatalogTransaction.java Normal file
View File

@ -0,0 +1,183 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.catalog;
import static com.google.common.base.Preconditions.checkArgument;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.protobuf.ByteString;
import com.pingcap.tikv.Snapshot;
import com.pingcap.tikv.codec.Codec.BytesCodec;
import com.pingcap.tikv.codec.Codec.IntegerCodec;
import com.pingcap.tikv.codec.CodecDataInput;
import com.pingcap.tikv.codec.CodecDataOutput;
import com.pingcap.tikv.codec.KeyUtils;
import com.pingcap.tikv.exception.TiClientInternalException;
import com.pingcap.tikv.kvproto.Kvrpcpb;
import com.pingcap.tikv.meta.TiDBInfo;
import com.pingcap.tikv.meta.TiTableInfo;
import com.pingcap.tikv.util.Pair;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import org.apache.log4j.Logger;
public class CatalogTransaction {
protected static final Logger logger = Logger.getLogger(Catalog.class);
private final Snapshot snapshot;
private final byte[] prefix;
private static final byte[] META_PREFIX = new byte[] {'m'};
private static final byte HASH_DATA_FLAG = 'h';
private static final byte STR_DATA_FLAG = 's';
private static ByteString KEY_DB = ByteString.copyFromUtf8("DBs");
private static ByteString KEY_TABLE = ByteString.copyFromUtf8("Table");
private static ByteString KEY_SCHEMA_VERSION = ByteString.copyFromUtf8("SchemaVersionKey");
private static final String ENCODED_DB_PREFIX = "DB";
public CatalogTransaction(Snapshot snapshot) {
this.snapshot = snapshot;
this.prefix = META_PREFIX;
}
private void encodeStringDataKey(CodecDataOutput cdo, byte[] key) {
cdo.write(prefix);
BytesCodec.writeBytes(cdo, key);
IntegerCodec.writeULong(cdo, STR_DATA_FLAG);
}
private void encodeHashDataKey(CodecDataOutput cdo, byte[] key, byte[] field) {
encodeHashDataKeyPrefix(cdo, key);
BytesCodec.writeBytes(cdo, field);
}
private void encodeHashDataKeyPrefix(CodecDataOutput cdo, byte[] key) {
cdo.write(prefix);
BytesCodec.writeBytes(cdo, key);
IntegerCodec.writeULong(cdo, HASH_DATA_FLAG);
}
private Pair<ByteString, ByteString> decodeHashDataKey(ByteString rawKey) {
checkArgument(
KeyUtils.hasPrefix(rawKey, ByteString.copyFrom(prefix)),
"invalid encoded hash data key prefix: " + new String(prefix));
CodecDataInput cdi = new CodecDataInput(rawKey.toByteArray());
cdi.skipBytes(prefix.length);
byte[] key = BytesCodec.readBytes(cdi);
long typeFlag = IntegerCodec.readULong(cdi);
if (typeFlag != HASH_DATA_FLAG) {
throw new TiClientInternalException("Invalid hash data flag: " + typeFlag);
}
byte[] field = BytesCodec.readBytes(cdi);
return Pair.create(ByteString.copyFrom(key), ByteString.copyFrom(field));
}
private ByteString hashGet(ByteString key, ByteString field) {
CodecDataOutput cdo = new CodecDataOutput();
encodeHashDataKey(cdo, key.toByteArray(), field.toByteArray());
return snapshot.get(cdo.toByteString());
}
private ByteString bytesGet(ByteString key) {
CodecDataOutput cdo = new CodecDataOutput();
encodeStringDataKey(cdo, key.toByteArray());
return snapshot.get(cdo.toByteString());
}
private List<Pair<ByteString, ByteString>> hashGetFields(ByteString key) {
CodecDataOutput cdo = new CodecDataOutput();
encodeHashDataKeyPrefix(cdo, key.toByteArray());
ByteString encodedKey = cdo.toByteString();
Iterator<Kvrpcpb.KvPair> iterator = snapshot.scan(encodedKey);
List<Pair<ByteString, ByteString>> fields = new ArrayList<>();
while (iterator.hasNext()) {
Kvrpcpb.KvPair kv = iterator.next();
if (!KeyUtils.hasPrefix(kv.getKey(), encodedKey)) {
break;
}
fields.add(Pair.create(decodeHashDataKey(kv.getKey()).second, kv.getValue()));
}
return fields;
}
private static ByteString encodeDatabaseID(long id) {
return ByteString.copyFrom(String.format("%s:%d", ENCODED_DB_PREFIX, id).getBytes());
}
public long getLatestSchemaVersion() {
ByteString versionBytes = bytesGet(KEY_SCHEMA_VERSION);
CodecDataInput cdi = new CodecDataInput(versionBytes.toByteArray());
return Long.parseLong(new String(cdi.toByteArray(), StandardCharsets.UTF_8));
}
public List<TiDBInfo> getDatabases() {
List<Pair<ByteString, ByteString>> fields = hashGetFields(KEY_DB);
ImmutableList.Builder<TiDBInfo> builder = ImmutableList.builder();
for (Pair<ByteString, ByteString> pair : fields) {
builder.add(parseFromJson(pair.second, TiDBInfo.class));
}
return builder.build();
}
public TiDBInfo getDatabase(long id) {
ByteString dbKey = encodeDatabaseID(id);
ByteString json = hashGet(KEY_DB, dbKey);
if (json == null || json.isEmpty()) {
return null;
}
return parseFromJson(json, TiDBInfo.class);
}
public List<TiTableInfo> getTables(long dbId) {
ByteString dbKey = encodeDatabaseID(dbId);
List<Pair<ByteString, ByteString>> fields = hashGetFields(dbKey);
ImmutableList.Builder<TiTableInfo> builder = ImmutableList.builder();
for (Pair<ByteString, ByteString> pair : fields) {
if (KeyUtils.hasPrefix(pair.first, KEY_TABLE)) {
builder.add(parseFromJson(pair.second, TiTableInfo.class));
}
}
return builder.build();
}
public static <T> T parseFromJson(ByteString json, Class<T> cls) {
Objects.requireNonNull(json, "json is null");
Objects.requireNonNull(cls, "cls is null");
logger.debug(String.format("Parse Json %s : %s", cls.getSimpleName(), json.toStringUtf8()));
ObjectMapper mapper = new ObjectMapper();
try {
return mapper.readValue(json.toStringUtf8(), cls);
} catch (JsonParseException | JsonMappingException e) {
String errMsg =
String.format(
"Invalid JSON value for Type %s: %s\n", cls.getSimpleName(), json.toStringUtf8());
throw new TiClientInternalException(errMsg, e);
} catch (Exception e1) {
throw new TiClientInternalException("Error parsing Json", e1);
}
}
}

656
src/main/java/com/pingcap/tikv/codec/Codec.java Normal file
View File

@ -0,0 +1,656 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.codec;
import static com.google.common.base.Preconditions.checkArgument;
import com.pingcap.tikv.exception.InvalidCodecFormatException;
import gnu.trove.list.array.TIntArrayList;
import java.math.BigDecimal;
import java.sql.Date;
import java.util.Arrays;
import org.joda.time.*;
public class Codec {
public static final int NULL_FLAG = 0;
public static final int BYTES_FLAG = 1;
public static final int COMPACT_BYTES_FLAG = 2;
public static final int INT_FLAG = 3;
public static final int UINT_FLAG = 4;
public static final int FLOATING_FLAG = 5;
public static final int DECIMAL_FLAG = 6;
public static final int DURATION_FLAG = 7;
public static final int VARINT_FLAG = 8;
public static final int UVARINT_FLAG = 9;
public static final int JSON_FLAG = 10;
public static final int MAX_FLAG = 250;
public static boolean isNullFlag(int flag) {
return flag == NULL_FLAG;
}
public static class IntegerCodec {
private static final long SIGN_MASK = ~Long.MAX_VALUE;
private static long flipSignBit(long v) {
return v ^ SIGN_MASK;
}
/**
* Encoding a long value to byte buffer with type flag at the beginning
*
* @param cdo For outputting data in bytes array
* @param lVal The data to encode
* @param comparable If the output should be memory comparable without decoding. In real TiDB
* use case, if used in Key encoding, we output memory comparable format otherwise not
*/
public static void writeLongFully(CodecDataOutput cdo, long lVal, boolean comparable) {
if (comparable) {
cdo.writeByte(INT_FLAG);
writeLong(cdo, lVal);
} else {
cdo.writeByte(VARINT_FLAG);
writeVarLong(cdo, lVal);
}
}
/**
* Encoding a unsigned long value to byte buffer with type flag at the beginning
*
* @param cdo For outputting data in bytes array
* @param lVal The data to encode, note that long is treated as unsigned
* @param comparable If the output should be memory comparable without decoding. In real TiDB
* use case, if used in Key encoding, we output memory comparable format otherwise not
*/
public static void writeULongFully(CodecDataOutput cdo, long lVal, boolean comparable) {
if (comparable) {
cdo.writeByte(UINT_FLAG);
writeULong(cdo, lVal);
} else {
cdo.writeByte(UVARINT_FLAG);
writeUVarLong(cdo, lVal);
}
}
/**
* Encode long value without type flag at the beginning The signed bit is flipped for memory
* comparable purpose
*
* @param cdo For outputting data in bytes array
* @param lVal The data to encode
*/
public static void writeLong(CodecDataOutput cdo, long lVal) {
cdo.writeLong(flipSignBit(lVal));
}
/**
* Encode long value without type flag at the beginning
*
* @param cdo For outputting data in bytes array
* @param lVal The data to encode
*/
public static void writeULong(CodecDataOutput cdo, long lVal) {
cdo.writeLong(lVal);
}
/**
* Encode var-length long, same as go's binary.PutVarint
*
* @param cdo For outputting data in bytes array
* @param value The data to encode
*/
static void writeVarLong(CodecDataOutput cdo, long value) {
long ux = value << 1;
if (value < 0) {
ux = ~ux;
}
writeUVarLong(cdo, ux);
}
/**
* Encode Data as var-length long, the same as go's binary.PutUvarint
*
* @param cdo For outputting data in bytes array
* @param value The data to encode
*/
static void writeUVarLong(CodecDataOutput cdo, long value) {
while ((value - 0x80) >= 0) {
cdo.writeByte((byte) value | 0x80);
value >>>= 7;
}
cdo.writeByte((byte) value);
}
/**
* Decode as signed long, assuming encoder flips signed bit for memory comparable
*
* @param cdi source of data
* @return decoded signed long value
*/
public static long readLong(CodecDataInput cdi) {
return flipSignBit(cdi.readLong());
}
public static long readPartialLong(CodecDataInput cdi) {
return flipSignBit(cdi.readPartialLong());
}
/**
* Decode as unsigned long without any binary manipulation
*
* @param cdi source of data
* @return decoded unsigned long value
*/
public static long readULong(CodecDataInput cdi) {
return cdi.readLong();
}
/**
* Decode as var-length long, the same as go's binary.Varint
*
* @param cdi source of data
* @return decoded signed long value
*/
public static long readVarLong(CodecDataInput cdi) {
long ux = readUVarLong(cdi);
long x = ux >>> 1;
if ((ux & 1) != 0) {
x = ~x;
}
return x;
}
/**
* Decode as var-length unsigned long, the same as go's binary.Uvarint
*
* @param cdi source of data
* @return decoded unsigned long value
*/
public static long readUVarLong(CodecDataInput cdi) {
long x = 0;
int s = 0;
for (int i = 0; !cdi.eof(); i++) {
long b = cdi.readUnsignedByte();
if ((b - 0x80) < 0) {
if (i > 9 || i == 9 && b > 1) {
throw new InvalidCodecFormatException("readUVarLong overflow");
}
return x | b << s;
}
x |= (b & 0x7f) << s;
s += 7;
}
throw new InvalidCodecFormatException("readUVarLong encountered unfinished data");
}
}
public static class BytesCodec {
private static final int GRP_SIZE = 8;
private static final byte[] PADS = new byte[GRP_SIZE];
private static final int MARKER = 0xFF;
private static final byte PAD = (byte) 0x0;
public static void writeBytesRaw(CodecDataOutput cdo, byte[] data) {
cdo.write(data);
}
public static void writeBytesFully(CodecDataOutput cdo, byte[] data) {
cdo.write(Codec.BYTES_FLAG);
BytesCodec.writeBytes(cdo, data);
}
// writeBytes guarantees the encoded value is in ascending order for comparison,
// encoding with the following rule:
// [group1][marker1]...[groupN][markerN]
// group is 8 bytes slice which is padding with 0.
// marker is `0xFF - padding 0 count`
// For example:
// [] -> [0, 0, 0, 0, 0, 0, 0, 0, 247]
// [1, 2, 3] -> [1, 2, 3, 0, 0, 0, 0, 0, 250]
// [1, 2, 3, 0] -> [1, 2, 3, 0, 0, 0, 0, 0, 251]
// [1, 2, 3, 4, 5, 6, 7, 8] -> [1, 2, 3, 4, 5, 6, 7, 8, 255, 0, 0, 0, 0, 0, 0, 0, 0, 247]
// Refer: https://github.com/facebook/mysql-5.6/wiki/MyRocks-record-format#memcomparable-format
public static void writeBytes(CodecDataOutput cdo, byte[] data) {
for (int i = 0; i <= data.length; i += GRP_SIZE) {
int remain = data.length - i;
int padCount = 0;
if (remain >= GRP_SIZE) {
cdo.write(data, i, GRP_SIZE);
} else {
padCount = GRP_SIZE - remain;
cdo.write(data, i, data.length - i);
cdo.write(PADS, 0, padCount);
}
cdo.write((byte) (MARKER - padCount));
}
}
public static void writeCompactBytesFully(CodecDataOutput cdo, byte[] data) {
cdo.write(Codec.COMPACT_BYTES_FLAG);
writeCompactBytes(cdo, data);
}
/**
* Write bytes in a compact form.
*
* @param cdo destination of data.
* @param data is value that will be written into cdo.
*/
static void writeCompactBytes(CodecDataOutput cdo, byte[] data) {
int length = data.length;
IntegerCodec.writeVarLong(cdo, length);
cdo.write(data);
}
// readBytes decodes bytes which is encoded by EncodeBytes before,
// returns the leftover bytes and decoded value if no error.
public static byte[] readBytes(CodecDataInput cdi) {
return readBytes(cdi, false);
}
public static byte[] readCompactBytes(CodecDataInput cdi) {
int size = (int) IntegerCodec.readVarLong(cdi);
return readCompactBytes(cdi, size);
}
private static byte[] readCompactBytes(CodecDataInput cdi, int size) {
byte[] data = new byte[size];
for (int i = 0; i < size; i++) {
data[i] = cdi.readByte();
}
return data;
}
private static byte[] readBytes(CodecDataInput cdi, boolean reverse) {
CodecDataOutput cdo = new CodecDataOutput();
while (true) {
byte[] groupBytes = new byte[GRP_SIZE + 1];
cdi.readFully(groupBytes, 0, GRP_SIZE + 1);
byte[] group = Arrays.copyOfRange(groupBytes, 0, GRP_SIZE);
int padCount;
int marker = Byte.toUnsignedInt(groupBytes[GRP_SIZE]);
if (reverse) {
padCount = marker;
} else {
padCount = MARKER - marker;
}
checkArgument(padCount <= GRP_SIZE);
int realGroupSize = GRP_SIZE - padCount;
cdo.write(group, 0, realGroupSize);
if (padCount != 0) {
byte padByte = PAD;
if (reverse) {
padByte = (byte) MARKER;
}
// Check validity of padding bytes.
for (int i = realGroupSize; i < group.length; i++) {
byte b = group[i];
checkArgument(padByte == b);
}
break;
}
}
byte[] bytes = cdo.toBytes();
if (reverse) {
for (int i = 0; i < bytes.length; i++) {
bytes[i] = (byte) ~bytes[i];
}
}
return bytes;
}
}
public static class RealCodec {
private static final long signMask = 0x8000000000000000L;
/**
* Decode as float
*
* @param cdi source of data
* @return decoded unsigned long value
*/
public static double readDouble(CodecDataInput cdi) {
long u = IntegerCodec.readULong(cdi);
if (u < 0) {
u &= Long.MAX_VALUE;
} else {
u = ~u;
}
return Double.longBitsToDouble(u);
}
private static long encodeDoubleToCmpLong(double val) {
long u = Double.doubleToRawLongBits(val);
if (val >= 0) {
u |= signMask;
} else {
u = ~u;
}
return u;
}
public static void writeDoubleFully(CodecDataOutput cdo, double val) {
cdo.writeByte(FLOATING_FLAG);
writeDouble(cdo, val);
}
/**
* Encoding a double value to byte buffer
*
* @param cdo For outputting data in bytes array
* @param val The data to encode
*/
public static void writeDouble(CodecDataOutput cdo, double val) {
IntegerCodec.writeULong(cdo, encodeDoubleToCmpLong(val));
}
}
public static class DecimalCodec {
/**
* read a decimal value from CodecDataInput
*
* @param cdi cdi is source data.
*/
public static BigDecimal readDecimal(CodecDataInput cdi) {
if (cdi.available() < 3) {
throw new IllegalArgumentException("insufficient bytes to read value");
}
// 64 should be larger enough for avoiding unnecessary growth.
TIntArrayList data = new TIntArrayList(64);
int precision = cdi.readUnsignedByte();
int frac = cdi.readUnsignedByte();
int length = precision + frac;
int curPos = cdi.size() - cdi.available();
for (int i = 0; i < length; i++) {
if (cdi.eof()) {
break;
}
data.add(cdi.readUnsignedByte());
}
MyDecimal dec = new MyDecimal();
int binSize = dec.fromBin(precision, frac, data.toArray());
cdi.mark(curPos + binSize);
cdi.reset();
return dec.toDecimal();
}
/**
* write a decimal value from CodecDataInput
*
* @param cdo cdo is destination data.
* @param dec is decimal value that will be written into cdo.
*/
static void writeDecimal(CodecDataOutput cdo, MyDecimal dec) {
int[] data = dec.toBin(dec.precision(), dec.frac());
cdo.writeByte(dec.precision());
cdo.writeByte(dec.frac());
for (int aData : data) {
cdo.writeByte(aData & 0xFF);
}
}
public static void writeDecimalFully(CodecDataOutput cdo, BigDecimal val) {
cdo.writeByte(DECIMAL_FLAG);
writeDecimal(cdo, val);
}
/**
* Encoding a double value to byte buffer
*
* @param cdo For outputting data in bytes array
* @param val The data to encode
*/
public static void writeDecimal(CodecDataOutput cdo, BigDecimal val) {
MyDecimal dec = new MyDecimal();
dec.fromString(val.toPlainString());
writeDecimal(cdo, dec);
}
}
public static class DateTimeCodec {
/**
* Encode a DateTime to a packed long converting to specific timezone
*
* @param dateTime dateTime that need to be encoded.
* @param tz timezone used for converting to localDateTime
* @return a packed long.
*/
static long toPackedLong(DateTime dateTime, DateTimeZone tz) {
LocalDateTime localDateTime = dateTime.withZone(tz).toLocalDateTime();
return toPackedLong(
localDateTime.getYear(),
localDateTime.getMonthOfYear(),
localDateTime.getDayOfMonth(),
localDateTime.getHourOfDay(),
localDateTime.getMinuteOfHour(),
localDateTime.getSecondOfMinute(),
localDateTime.getMillisOfSecond() * 1000);
}
/**
* Encode a date/time parts to a packed long.
*
* @return a packed long.
*/
static long toPackedLong(
int year, int month, int day, int hour, int minute, int second, int micro) {
long ymd = (year * 13 + month) << 5 | day;
long hms = hour << 12 | minute << 6 | second;
return ((ymd << 17 | hms) << 24) | micro;
}
/**
* Read datetime from packed Long which contains all parts of a datetime namely, year, month,
* day and hour, min and sec, millisec. The original representation does not indicate any
* timezone information In Timestamp type, it should be interpreted as UTC while in DateType it
* is interpreted as local timezone
*
* @param packed long value that packs date / time parts
* @param tz timezone to interpret datetime parts
* @return decoded DateTime using provided timezone
*/
static DateTime fromPackedLong(long packed, DateTimeZone tz) {
// TODO: As for JDBC behavior, it can be configured to "round" or "toNull"
// for now we didn't pass in session so we do a toNull behavior
if (packed == 0) {
return null;
}
long ymdhms = packed >> 24;
long ymd = ymdhms >> 17;
int day = (int) (ymd & ((1 << 5) - 1));
long ym = ymd >> 5;
int month = (int) (ym % 13);
int year = (int) (ym / 13);
int hms = (int) (ymdhms & ((1 << 17) - 1));
int second = hms & ((1 << 6) - 1);
int minute = (hms >> 6) & ((1 << 6) - 1);
int hour = hms >> 12;
int microsec = (int) (packed % (1 << 24));
try {
return new DateTime(year, month, day, hour, minute, second, microsec / 1000, tz);
} catch (IllegalInstantException e) {
LocalDateTime localDateTime =
new LocalDateTime(year, month, day, hour, minute, second, microsec / 1000);
DateTime dt = localDateTime.toLocalDate().toDateTimeAtStartOfDay(tz);
long millis = dt.getMillis() + localDateTime.toLocalTime().getMillisOfDay();
return new DateTime(millis, tz);
}
}
/**
* Encode DateTime as packed long converting into specified timezone All timezone conversion
* should be done beforehand
*
* @param cdo encoding output
* @param dateTime value to encode
* @param tz timezone used to converting local time
*/
public static void writeDateTimeFully(CodecDataOutput cdo, DateTime dateTime, DateTimeZone tz) {
long val = DateTimeCodec.toPackedLong(dateTime, tz);
IntegerCodec.writeULongFully(cdo, val, true);
}
/**
* Encode DateTime as packed long converting into specified timezone All timezone conversion
* should be done beforehand The encoded value has no data type flag
*
* @param cdo encoding output
* @param dateTime value to encode
* @param tz timezone used to converting local time
*/
public static void writeDateTimeProto(CodecDataOutput cdo, DateTime dateTime, DateTimeZone tz) {
long val = DateTimeCodec.toPackedLong(dateTime, tz);
IntegerCodec.writeULong(cdo, val);
}
/**
* Read datetime from packed Long encoded as unsigned var-len integer converting into specified
* timezone
*
* @see DateTimeCodec#fromPackedLong(long, DateTimeZone)
* @param cdi codec buffer input
* @param tz timezone to interpret datetime parts
* @return decoded DateTime using provided timezone
*/
public static DateTime readFromUVarInt(CodecDataInput cdi, DateTimeZone tz) {
return DateTimeCodec.fromPackedLong(IntegerCodec.readUVarLong(cdi), tz);
}
/**
* Read datetime from packed Long as unsigned fixed-len integer
*
* @see DateTimeCodec#fromPackedLong(long, DateTimeZone)
* @param cdi codec buffer input
* @param tz timezone to interpret datetime parts
* @return decoded DateTime using provided timezone
*/
public static DateTime readFromUInt(CodecDataInput cdi, DateTimeZone tz) {
return DateTimeCodec.fromPackedLong(IntegerCodec.readULong(cdi), tz);
}
}
public static class DateCodec {
/**
* Encode a UTC Date to a packed long converting to specific timezone
*
* @param date date that need to be encoded.
* @param tz timezone used for converting to localDate
* @return a packed long.
*/
static long toPackedLong(Date date, DateTimeZone tz) {
return toPackedLong(date.getTime(), tz);
}
static long toPackedLong(long utcMillsTs, DateTimeZone tz) {
LocalDate date = new LocalDate(utcMillsTs, tz);
return toPackedLong(date);
}
static long toPackedLong(LocalDate date) {
return Codec.DateCodec.toPackedLong(
date.getYear(), date.getMonthOfYear(), date.getDayOfMonth());
}
/**
* Encode a date part to a packed long.
*
* @return a packed long.
*/
static long toPackedLong(int year, int month, int day) {
long ymd = (year * 13 + month) << 5 | day;
return ymd << 41;
}
static LocalDate fromPackedLong(long packed) {
// TODO: As for JDBC behavior, it can be configured to "round" or "toNull"
// for now we didn't pass in session so we do a toNull behavior
if (packed == 0) {
return null;
}
long ymd = packed >> 41;
int day = (int) (ymd & ((1 << 5) - 1));
long ym = ymd >> 5;
int month = (int) (ym % 13);
int year = (int) (ym / 13);
return new LocalDate(year, month, day, null);
}
/**
* Encode Date as packed long converting into specified timezone All timezone conversion should
* be done beforehand
*
* @param cdo encoding output
* @param date value to encode
* @param tz timezone used to converting local time
*/
public static void writeDateFully(CodecDataOutput cdo, Date date, DateTimeZone tz) {
long val = DateCodec.toPackedLong(date, tz);
IntegerCodec.writeULongFully(cdo, val, true);
}
/**
* Encode Date as packed long converting into specified timezone All timezone conversion should
* be done beforehand The encoded value has no data type flag
*
* @param cdo encoding output
* @param date value to encode
* @param tz timezone used to converting local time
*/
public static void writeDateProto(CodecDataOutput cdo, Date date, DateTimeZone tz) {
long val = DateCodec.toPackedLong(date, tz);
IntegerCodec.writeULong(cdo, val);
}
/**
* Read date from packed Long encoded as unsigned var-len integer converting into specified
* timezone
*
* @see DateCodec#fromPackedLong(long)
* @param cdi codec buffer input
* @return decoded DateTime using provided timezone
*/
public static LocalDate readFromUVarInt(CodecDataInput cdi) {
return DateCodec.fromPackedLong(IntegerCodec.readUVarLong(cdi));
}
/**
* Read date from packed Long as unsigned fixed-len integer
*
* @see DateCodec#fromPackedLong(long)
* @param cdi codec buffer input
* @return decoded DateTime using provided timezone
*/
public static LocalDate readFromUInt(CodecDataInput cdi) {
return DateCodec.fromPackedLong(IntegerCodec.readULong(cdi));
}
}
}

321
src/main/java/com/pingcap/tikv/codec/CodecDataInput.java Normal file
View File

@ -0,0 +1,321 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.codec;
import com.google.protobuf.ByteString;
import java.io.*;
public class CodecDataInput implements DataInput {
/**
* An copy of ByteArrayInputStream without synchronization for faster decode.
*
* @see ByteArrayInputStream
*/
private class UnSyncByteArrayInputStream extends InputStream {
protected byte buf[];
protected int pos;
protected int mark = 0;
protected int count;
UnSyncByteArrayInputStream(byte buf[]) {
this.buf = buf;
this.pos = 0;
this.count = buf.length;
}
public UnSyncByteArrayInputStream(byte buf[], int offset, int length) {
this.buf = buf;
this.pos = offset;
this.count = Math.min(offset + length, buf.length);
this.mark = offset;
}
public int read() {
return (pos < count) ? (buf[pos++] & 0xff) : -1;
}
public int read(byte b[], int off, int len) {
if (b == null) {
throw new NullPointerException();
} else if (off < 0 || len < 0 || len > b.length - off) {
throw new IndexOutOfBoundsException();
}
if (pos >= count) {
return -1;
}
int avail = count - pos;
if (len > avail) {
len = avail;
}
if (len <= 0) {
return 0;
}
System.arraycopy(buf, pos, b, off, len);
pos += len;
return len;
}
public long skip(long n) {
long k = count - pos;
if (n < k) {
k = n < 0 ? 0 : n;
}
pos += k;
return k;
}
public int available() {
return count - pos;
}
public boolean markSupported() {
return true;
}
public void mark(int readAheadLimit) {
mark = pos;
}
public void reset() {
pos = mark;
}
public void close() throws IOException {}
}
private final DataInputStream inputStream;
private final UnSyncByteArrayInputStream backingStream;
private final byte[] backingBuffer;
public CodecDataInput(ByteString data) {
this(data.toByteArray());
}
public CodecDataInput(byte[] buf) {
backingBuffer = buf;
// MyDecimal usually will consume more bytes. If this happened,
// we need have a mechanism to reset backingStream.
// User mark first and then reset it later can do the trick.
backingStream =
new UnSyncByteArrayInputStream(buf) {
@Override
public void mark(int givenPos) {
mark = givenPos;
}
};
inputStream = new DataInputStream(backingStream);
}
@Override
public void readFully(byte[] b) {
try {
inputStream.readFully(b);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void readFully(byte[] b, int off, int len) {
try {
inputStream.readFully(b, off, len);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public int skipBytes(int n) {
try {
return inputStream.skipBytes(n);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public boolean readBoolean() {
try {
return inputStream.readBoolean();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public byte readByte() {
try {
return inputStream.readByte();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public int readUnsignedByte() {
try {
return inputStream.readUnsignedByte();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public short readShort() {
try {
return inputStream.readShort();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public int readUnsignedShort() {
try {
return inputStream.readUnsignedShort();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public int readPartialUnsignedShort() {
try {
byte readBuffer[] = new byte[2];
inputStream.read(readBuffer, 0, 2);
return ((readBuffer[0] & 0xff) << 8) + ((readBuffer[1] & 0xff) << 0);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public char readChar() {
try {
return inputStream.readChar();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public int readInt() {
try {
return inputStream.readInt();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public long readLong() {
try {
return inputStream.readLong();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public final long readPartialLong() {
try {
byte readBuffer[] = new byte[8];
inputStream.read(readBuffer, 0, 8);
return (((long) readBuffer[0] << 56)
+ ((long) (readBuffer[1] & 255) << 48)
+ ((long) (readBuffer[2] & 255) << 40)
+ ((long) (readBuffer[3] & 255) << 32)
+ ((long) (readBuffer[4] & 255) << 24)
+ ((readBuffer[5] & 255) << 16)
+ ((readBuffer[6] & 255) << 8)
+ ((readBuffer[7] & 255) << 0));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public float readFloat() {
try {
return inputStream.readFloat();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public double readDouble() {
try {
return inputStream.readDouble();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public String readLine() {
try {
return inputStream.readLine();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public String readUTF() {
try {
return inputStream.readUTF();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public int peekByte() {
mark(currentPos());
int b = readByte() & 0xFF;
reset();
return b;
}
public int currentPos() {
return size() - available();
}
public void mark(int givenPos) {
this.backingStream.mark(givenPos);
}
public void reset() {
this.backingStream.reset();
}
public boolean eof() {
return backingStream.available() == 0;
}
public int size() {
return backingBuffer.length;
}
public int available() {
return backingStream.available();
}
public byte[] toByteArray() {
return backingBuffer;
}
}

176
src/main/java/com/pingcap/tikv/codec/CodecDataOutput.java Normal file
View File

@ -0,0 +1,176 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.codec;
import com.google.protobuf.ByteString;
import java.io.ByteArrayOutputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
// A trivial implementation supposed to be replaced
public class CodecDataOutput implements DataOutput {
private DataOutputStream s;
// TODO: Switch to ByteBuffer if possible, or a chain of ByteBuffer
private ByteArrayOutputStream byteArray;
public CodecDataOutput() {
byteArray = new ByteArrayOutputStream();
s = new DataOutputStream(byteArray);
}
public CodecDataOutput(int size) {
byteArray = new ByteArrayOutputStream(size);
s = new DataOutputStream(byteArray);
}
@Override
public void write(int b) {
try {
s.write(b);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void write(byte b[]) {
try {
s.write(b);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void write(byte[] b, int off, int len) {
try {
s.write(b, off, len);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void writeBoolean(boolean v) {
try {
s.writeBoolean(v);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void writeByte(int v) {
try {
s.writeByte(v);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void writeShort(int v) {
try {
s.writeShort(v);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void writeChar(int v) {
try {
s.writeChar(v);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void writeInt(int v) {
try {
s.writeInt(v);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void writeLong(long v) {
try {
s.writeLong(v);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void writeFloat(float v) {
try {
s.writeFloat(v);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void writeDouble(double v) {
try {
s.writeDouble(v);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void writeBytes(String v) {
try {
s.writeBytes(v);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void writeChars(String v) {
try {
s.writeChars(v);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void writeUTF(String v) {
try {
s.writeUTF(v);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public byte[] toBytes() {
return byteArray.toByteArray();
}
public ByteString toByteString() {
return ByteString.copyFrom(byteArray.toByteArray());
}
public void reset() {
this.byteArray.reset();
}
}

54
src/main/java/com/pingcap/tikv/codec/KeyUtils.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.codec;
import com.google.common.primitives.UnsignedBytes;
import com.google.protobuf.ByteString;
import com.pingcap.tikv.kvproto.Coprocessor;
public class KeyUtils {
public static String formatBytes(byte[] bytes) {
if (bytes == null) return "null";
StringBuilder sb = new StringBuilder();
for (int i = 0; i < bytes.length; i++) {
int unsignedByte = UnsignedBytes.toInt(bytes[i]);
sb.append(unsignedByte);
if (i != bytes.length - 1) {
sb.append(",");
}
}
return sb.toString();
}
public static String formatBytes(ByteString bytes) {
if (bytes == null) return "null";
return formatBytes(bytes.toByteArray());
}
public static String formatBytes(Coprocessor.KeyRange keyRange) {
return "[[" + formatBytes(keyRange.getStart()) + "], [" + formatBytes(keyRange.getEnd()) + "])";
}
public static boolean hasPrefix(ByteString str, ByteString prefix) {
for (int i = 0; i < prefix.size(); i++) {
if (str.byteAt(i) != prefix.byteAt(i)) {
return false;
}
}
return true;
}
}

829
src/main/java/com/pingcap/tikv/codec/MyDecimal.java Normal file
View File

@ -0,0 +1,829 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.codec;
import com.google.common.annotations.VisibleForTesting;
import java.math.BigDecimal;
import java.util.Arrays;
// TODO: We shouldn't allow empty MyDecimal
// TODO: It seems MyDecimal to BigDecimal is very slow
public class MyDecimal {
// how many digits that a word has
private static final int digitsPerWord = 9;
// MyDecimal can holds at most 9 words.
private static final int wordBufLen = 9;
// A word is 4 bytes int
private static final int wordSize = 4;
private static final int ten0 = 1;
private static final int ten1 = 10;
private static final int ten2 = 100;
private static final int ten3 = 1000;
private static final int ten4 = 10000;
private static final int ten5 = 100000;
private static final int ten6 = 1000000;
private static final int ten7 = 10000000;
private static final int ten8 = 100000000;
private static final int ten9 = 1000000000;
private static final int digMask = ten8;
private static final int wordBase = ten9;
private static final int wordMax = wordBase - 1;
private static final int[] powers10 =
new int[] {ten0, ten1, ten2, ten3, ten4, ten5, ten6, ten7, ten8, ten9};
// A MyDecimal holds 9 words.
private static final int maxWordBufLen = 9;
private static final int maxFraction = 30;
private static final int[] dig2bytes = new int[] {0, 1, 1, 2, 2, 3, 3, 4, 4, 4};
// The following are fields of MyDecimal
private int digitsInt;
private int digitsFrac;
private int resultFrac;
private boolean negative;
private int[] wordBuf = new int[maxWordBufLen];
/*
* Returns total precision of this decimal. Basically, it is sum of digitsInt and digitsFrac. But there
* are some special cases need to be token care of such as 000.001.
* Precision reflects the actual effective precision without leading zero
*/
public int precision() {
int frac = this.digitsFrac;
int digitsInt =
this.removeLeadingZeros()[
1]; /*this function return an array and the second element is digitsInt*/
int precision = digitsInt + frac;
// if no precision, it is just 0.
if (precision == 0) {
precision = 1;
}
return precision;
}
/**
* Returns fraction digits that counts how many digits after ".". frac() reflects the actual
* effective fraction without trailing zero
*/
public int frac() {
return digitsFrac;
}
/**
* Parses a decimal value from a string
*
* @param value an double value
*/
public void fromDecimal(double value) {
String s = Double.toString(value);
this.fromString(s);
}
/**
* Parses a decimal from binary string for given precision and frac.
*
* @param precision precision specifies total digits that this decimal will be..
* @param frac frac specifies how many fraction digits
* @param bin bin is binary string which represents a decimal value.
*/
public int fromBin(int precision, int frac, int[] bin) {
if (bin.length == 0) {
throw new IllegalArgumentException("Bad Float Number to parse");
}
int digitsInt = precision - frac;
int wordsInt = digitsInt / digitsPerWord;
int leadingDigits = digitsInt - wordsInt * digitsPerWord;
int wordsFrac = frac / digitsPerWord;
int trailingDigits = frac - wordsFrac * digitsPerWord;
int wordsIntTo = wordsInt;
if (leadingDigits > 0) {
wordsIntTo++;
}
int wordsFracTo = wordsFrac;
if (trailingDigits > 0) {
wordsFracTo++;
}
int binIdx = 0;
int mask = -1;
int sign = bin[binIdx] & 0x80;
if (sign > 0) {
mask = 0;
}
int binSize = decimalBinSize(precision, frac);
int[] dCopy;
dCopy = Arrays.copyOf(bin, binSize);
dCopy[0] ^= 0x80;
bin = dCopy;
int oldWordsIntTo = wordsIntTo;
boolean overflow = false;
boolean truncated = false;
if (wordsIntTo + wordsFracTo > wordBufLen) {
if (wordsIntTo > wordBufLen) {
wordsIntTo = wordBufLen;
wordsFracTo = 0;
overflow = true;
} else {
wordsIntTo = wordsInt;
wordsFracTo = wordBufLen - wordsInt;
truncated = true;
}
}
if (overflow || truncated) {
if (wordsIntTo < oldWordsIntTo) {
binIdx += dig2bytes[leadingDigits] + (wordsInt - wordsIntTo) * wordSize;
} else {
trailingDigits = 0;
wordsFrac = wordsFracTo;
}
}
this.negative = mask != 0;
this.digitsInt = (byte) (wordsInt * digitsPerWord + leadingDigits);
this.digitsFrac = (byte) (wordsFrac * digitsPerWord + trailingDigits);
int wordIdx = 0;
if (leadingDigits > 0) {
int i = dig2bytes[leadingDigits];
int x = readWord(bin, i, binIdx);
binIdx += i;
this.wordBuf[wordIdx] = (x ^ mask) > 0 ? x ^ mask : (x ^ mask) & 0xFF;
if (this.wordBuf[wordIdx] >= powers10[leadingDigits + 1]) {
throw new IllegalArgumentException("BadNumber");
}
if (this.wordBuf[wordIdx] != 0) {
wordIdx++;
} else {
this.digitsInt -= leadingDigits;
}
}
for (int stop = binIdx + wordsInt * wordSize; binIdx < stop; binIdx += wordSize) {
this.wordBuf[wordIdx] = (readWord(bin, 4, binIdx) ^ mask);
if (this.wordBuf[wordIdx] > wordMax) {
throw new IllegalArgumentException("BadNumber");
}
if (wordIdx > 0 || this.wordBuf[wordIdx] != 0) {
wordIdx++;
} else {
this.digitsInt -= digitsPerWord;
}
}
for (int stop = binIdx + wordsFrac * wordSize; binIdx < stop; binIdx += wordSize) {
int x = readWord(bin, 4, binIdx);
this.wordBuf[wordIdx] = (x ^ mask) > 0 ? x ^ mask : (x ^ mask) & 0xFF;
if (this.wordBuf[wordIdx] > wordMax) {
throw new IllegalArgumentException("BadNumber");
}
wordIdx++;
}
if (trailingDigits > 0) {
int i = dig2bytes[trailingDigits];
int x = readWord(bin, i, binIdx);
this.wordBuf[wordIdx] =
((x ^ mask) > 0 ? x ^ mask : (x ^ mask) & 0xFF)
* powers10[digitsPerWord - trailingDigits];
if (this.wordBuf[wordIdx] > wordMax) {
throw new IllegalArgumentException("BadNumber");
}
wordIdx++;
}
this.resultFrac = frac;
return binSize;
}
/** Returns a double value from MyDecimal instance. */
public BigDecimal toDecimal() {
return new BigDecimal(toString());
}
public double toDouble() {
return Float.parseFloat(toString());
}
/** Truncates any prefix zeros such as 00.001. After this, digitsInt is truncated from 2 to 0. */
private int[] removeLeadingZeros() {
int wordIdx = 0;
int digitsInt = this.digitsInt;
int i = ((digitsInt - 1) % digitsPerWord) + 1;
for (; digitsInt > 0 && this.wordBuf[wordIdx] == 0; ) {
digitsInt -= i;
i = digitsPerWord;
wordIdx++;
}
if (digitsInt > 0) {
digitsInt -= countLeadingZeroes((digitsInt - 1) % digitsPerWord, this.wordBuf[wordIdx]);
} else {
digitsInt = 0;
}
int[] res = new int[2];
res[0] = wordIdx;
res[1] = digitsInt;
return res;
}
/**
* Counts the number of digits of prefix zeors. For 00.001, it reutrns two.
*
* @param i i is index for getting powers10.
* @param word word is a integer.
*/
private int countLeadingZeroes(int i, int word) {
int leading = 0;
for (; word < powers10[i]; ) {
i--;
leading++;
}
return leading;
}
private int min(int a, int b) {
if (a > b) return b;
else return a;
}
/** Returns size of word for a give value with number of digits */
private int digitsToWords(int digits) {
return (digits + digitsPerWord - 1) / digitsPerWord;
}
/**
* Reads a word from a array at given size.
*
* @param b b is source data of unsigned byte as int[]
* @param size is word size which can be used in switch statement.
* @param start start indicates the where start to read.
*/
@VisibleForTesting
public static int readWord(int[] b, int size, int start) {
int x = 0;
switch (size) {
case 1:
x = (byte) b[start];
break;
case 2:
x = (((byte) b[start]) << 8) + (b[start + 1] & 0xFF);
break;
case 3:
int sign = b[start] & 128;
if (sign > 0) {
x = 0xFF << 24 | (b[start] << 16) | (b[start + 1] << 8) | (b[start + 2]);
} else {
x = b[start] << 16 | (b[start + 1] << 8) | b[start + 2];
}
break;
case 4:
x = b[start + 3] + (b[start + 2] << 8) + (b[start + 1] << 16) + (b[start] << 24);
break;
}
return x;
}
/**
* parser a decimal value from a string.
*
* @param s s is a decimal in string form.
*/
@VisibleForTesting
public void fromString(String s) {
char[] sCharArray = s.toCharArray();
fromCharArray(sCharArray);
}
// helper function for fromString
private void fromCharArray(char[] str) {
int startIdx = 0;
// found first character is not space and start from here
for (; startIdx < str.length; startIdx++) {
if (!Character.isSpaceChar(str[startIdx])) {
break;
}
}
if (str.length == 0) {
throw new IllegalArgumentException("BadNumber");
}
// skip sign and record where digits start from
// [-, 1, 2, 3]
// [+, 1, 2, 3]
// for +/-, we need skip them and record sign information into negative field.
switch (str[startIdx]) {
case '-':
this.negative = true;
startIdx++;
break;
case '+':
startIdx++;
break;
}
int strIdx = startIdx;
for (; strIdx < str.length && Character.isDigit(str[strIdx]); ) {
strIdx++;
}
// we initialize strIdx in case of sign notation, here we need substract startIdx from strIdx
// casue strIdx is used for counting the number of digits.
int digitsInt = strIdx - startIdx;
int digitsFrac;
int endIdx;
if (strIdx < str.length && str[strIdx] == '.') {
endIdx = strIdx + 1;
// detect where is the end index of this char array.
for (; endIdx < str.length && Character.isDigit(str[endIdx]); ) {
endIdx++;
}
digitsFrac = endIdx - strIdx - 1;
} else {
digitsFrac = 0;
endIdx = strIdx;
}
if (digitsInt + digitsFrac == 0) {
throw new IllegalArgumentException("BadNumber");
}
int wordsInt = digitsToWords(digitsInt);
int wordsFrac = digitsToWords(digitsFrac);
// TODO the following code are fixWordCntError such as overflow and truncated error
boolean overflow = false;
boolean truncated = false;
if (wordsInt + wordsFrac > wordBufLen) {
if (wordsInt > wordBufLen) {
wordsInt = wordBufLen;
wordsFrac = 0;
overflow = true;
} else {
wordsFrac = wordBufLen - wordsInt;
truncated = true;
}
}
if (overflow || truncated) {
digitsFrac = wordsFrac * digitsPerWord;
if (overflow) {
digitsInt = wordsInt * digitsPerWord;
}
}
this.digitsInt = digitsInt;
this.digitsFrac = digitsFrac;
int wordIdx = wordsInt;
int strIdxTmp = strIdx;
int word = 0;
int innerIdx = 0;
for (; digitsInt > 0; ) {
digitsInt--;
strIdx--;
word += (str[strIdx] - '0') * powers10[innerIdx];
innerIdx++;
if (innerIdx == digitsPerWord) {
wordIdx--;
this.wordBuf[wordIdx] = word;
word = 0;
innerIdx = 0;
}
}
if (innerIdx != 0) {
wordIdx--;
this.wordBuf[wordIdx] = word;
}
wordIdx = wordsInt;
strIdx = strIdxTmp;
word = 0;
innerIdx = 0;
for (; digitsFrac > 0; ) {
digitsFrac--;
strIdx++;
word = (str[strIdx] - '0') + word * 10;
innerIdx++;
if (innerIdx == digitsPerWord) {
this.wordBuf[wordIdx] = word;
wordIdx++;
word = 0;
innerIdx = 0;
}
}
if (innerIdx != 0) {
this.wordBuf[wordIdx] = word * powers10[digitsPerWord - innerIdx];
}
// this is -0000 is just 0.
boolean allZero = true;
for (int i = 0; i < wordBufLen; i++) {
if (this.wordBuf[i] != 0) {
allZero = false;
break;
}
}
if (allZero) {
this.negative = false;
}
this.resultFrac = this.digitsFrac;
}
// parser a string to a int.
private int strToLong(String str) {
str = str.trim();
if (str.isEmpty()) {
return 0;
}
boolean negative = false;
int i = 0;
if (str.charAt(i) == '-') {
negative = true;
i++;
} else if (str.charAt(i) == '+') {
i++;
}
int r = 0;
for (; i < str.length(); i++) {
if (!Character.isDigit(str.charAt(i))) {
break;
}
r = r * 10 + (str.charAt(i) - '0');
}
if (negative) {
r = -r;
}
return r;
}
// Returns a decimal string.
public String toString() {
char[] str;
int digitsFrac = this.digitsFrac;
int[] res = removeLeadingZeros();
int wordStartIdx = res[0];
int digitsInt = res[1];
if (digitsInt + digitsFrac == 0) {
digitsInt = 1;
wordStartIdx = 0;
}
int digitsIntLen = digitsInt;
if (digitsIntLen == 0) {
digitsIntLen = 1;
}
int digitsFracLen = digitsFrac;
int length = digitsIntLen + digitsFracLen;
if (this.negative) {
length++;
}
if (digitsFrac > 0) {
length++;
}
str = new char[length];
int strIdx = 0;
if (this.negative) {
str[strIdx] = '-';
strIdx++;
}
int fill = 0;
if (digitsFrac > 0) {
int fracIdx = strIdx + digitsIntLen;
fill = digitsFracLen - digitsFrac;
int wordIdx = wordStartIdx + digitsToWords(digitsInt);
str[fracIdx] = '.';
fracIdx++;
for (; digitsFrac > 0; digitsFrac -= digitsPerWord) {
int x = this.wordBuf[wordIdx];
wordIdx++;
for (int i = min(digitsFrac, digitsPerWord); i > 0; i--) {
int y = x / digMask;
str[fracIdx] = (char) (y + '0');
fracIdx++;
x -= y * digMask;
x *= 10;
}
}
for (; fill > 0; fill--) {
str[fracIdx] = '0';
fracIdx++;
}
}
fill = digitsIntLen - digitsInt;
if (digitsInt == 0) {
fill--; /* symbol 0 before digital point */
}
for (; fill > 0; fill--) {
str[strIdx] = '0';
strIdx++;
}
if (digitsInt > 0) {
strIdx += digitsInt;
int wordIdx = wordStartIdx + digitsToWords(digitsInt);
for (; digitsInt > 0; digitsInt -= digitsPerWord) {
wordIdx--;
int x = this.wordBuf[wordIdx];
for (int i = min(digitsInt, digitsPerWord); i > 0; i--) {
int y = x / 10;
strIdx--;
str[strIdx] = (char) ('0' + (x - y * 10));
x = y;
}
}
} else {
str[strIdx] = '0';
}
return new String(str);
}
private int stringSize() {
return digitsInt + digitsFrac + 3;
}
public long toLong() {
long x = 0;
int wordIdx = 0;
for (int i = this.digitsInt; i > 0; i -= digitsPerWord) {
/*
Attention: trick!
we're calculating -|from| instead of |from| here
because |LONGLONG_MIN| > LONGLONG_MAX
so we can convert -9223372036854775808 correctly
*/
long y = x;
x = x * wordBase - (long) this.wordBuf[wordIdx];
wordIdx++;
if (y < Long.MIN_VALUE / wordBase || x > y) {
/*
the decimal is bigger than any possible integer
return border integer depending on the sign
*/
if (this.negative) {
return Long.MIN_VALUE;
}
return Long.MAX_VALUE;
}
}
/* boundary case: 9223372036854775808 */
if (!this.negative && x == Long.MIN_VALUE) {
return Long.MAX_VALUE;
}
if (!this.negative) {
x = -x;
}
for (int i = this.digitsFrac; i > 0; i -= digitsPerWord) {
if (this.wordBuf[wordIdx] != 0) {
return x;
}
wordIdx++;
}
return x;
}
// decimalBinSize returns the size of array to hold a binary representation of a decimal.
private int decimalBinSize(int precision, int frac) {
int digitsInt = precision - frac;
int wordsInt = digitsInt / digitsPerWord;
int wordsFrac = frac / digitsPerWord;
int xInt = digitsInt - wordsInt * digitsPerWord;
int xFrac = frac - wordsFrac * digitsPerWord;
return wordsInt * wordSize + dig2bytes[xInt] + wordsFrac * wordSize + dig2bytes[xFrac];
}
/**
* ToBin converts decimal to its binary fixed-length representation two representations of the
* same length can be compared with memcmp with the correct -1/0/+1 result
*
* <p>PARAMS precision/frac - if precision is 0, internal value of the decimal will be used, then
* the encoded value is not memory comparable.
*
* <p>NOTE the buffer is assumed to be of the size decimalBinSize(precision, frac)
*
* <p>RETURN VALUE bin - binary value errCode - eDecOK/eDecTruncate/eDecOverflow
*
* <p>DESCRIPTION for storage decimal numbers are converted to the "binary" format.
*
* <p>This format has the following properties: 1. length of the binary representation depends on
* the {precision, frac} as provided by the caller and NOT on the digitsInt/digitsFrac of the
* decimal to convert. 2. binary representations of the same {precision, frac} can be compared
* with memcmp - with the same result as DecimalCompare() of the original decimals (not taking
* into account possible precision loss during conversion).
*
* <p>This binary format is as follows: 1. First the number is converted to have a requested
* precision and frac. 2. Every full digitsPerWord digits of digitsInt part are stored in 4 bytes
* as is 3. The first digitsInt % digitesPerWord digits are stored in the reduced number of bytes
* (enough bytes to store this number of digits - see dig2bytes) 4. same for frac - full word are
* stored as is, the last frac % digitsPerWord digits - in the reduced number of bytes. 5. If the
* number is negative - every byte is inversed. 5. The very first bit of the resulting byte array
* is inverted (because memcmp compares unsigned bytes, see property 2 above)
*
* <p>Example:
*
* <p>1234567890.1234
*
* <p>internally is represented as 3 words
*
* <p>1 234567890 123400000
*
* <p>(assuming we want a binary representation with precision=14, frac=4) in hex it's
*
* <p>00-00-00-01 0D-FB-38-D2 07-5A-EF-40
*
* <p>now, middle word is full - it stores 9 decimal digits. It goes into binary representation as
* is:
*
* <p>........... 0D-FB-38-D2 ............
*
* <p>First word has only one decimal digit. We can store one digit in one byte, no need to waste
* four:
*
* <p>01 0D-FB-38-D2 ............
*
* <p>now, last word. It's 123400000. We can store 1234 in two bytes:
*
* <p>01 0D-FB-38-D2 04-D2
*
* <p>So, we've packed 12 bytes number in 7 bytes. And now we invert the highest bit to get the
* final result:
*
* <p>81 0D FB 38 D2 04 D2
*
* <p>And for -1234567890.1234 it would be
*
* <p>7E F2 04 C7 2D FB 2D return a int array which represents a decimal value.
*
* @param precision precision for decimal value.
* @param frac fraction for decimal value.
*/
public int[] toBin(int precision, int frac) {
if (precision > digitsPerWord * maxWordBufLen
|| precision < 0
|| frac > maxFraction
|| frac < 0) {
throw new IllegalArgumentException("BadNumber");
}
int mask = 0;
if (this.negative) {
mask = -1;
}
int digitsInt = precision - frac;
int wordsInt = digitsInt / digitsPerWord;
int leadingDigits = digitsInt - wordsInt * digitsPerWord;
int wordsFrac = frac / digitsPerWord;
int trailingDigits = frac - wordsFrac * digitsPerWord;
// this should be one of 0, 1, 2, 3, 4
int wordsFracFrom = this.digitsFrac / digitsPerWord;
int trailingDigitsFrom = this.digitsFrac - wordsFracFrom * digitsPerWord;
int intSize = wordsInt * wordSize + dig2bytes[leadingDigits];
int fracSize = wordsFrac * wordSize + dig2bytes[trailingDigits];
int fracSizeFrom = wordsFracFrom * wordSize + dig2bytes[trailingDigitsFrom];
int originIntSize = intSize;
int originFracSize = fracSize;
int[] bin = new int[intSize + fracSize];
int binIdx = 0;
int[] res = this.removeLeadingZeros();
int wordIdxFrom = res[0];
int digitsIntFrom = res[1];
if (digitsIntFrom + fracSizeFrom == 0) {
mask = 0;
digitsInt = 1;
}
int wordsIntFrom = digitsIntFrom / digitsPerWord;
int leadingDigitsFrom = digitsIntFrom - wordsIntFrom * digitsPerWord;
int iSizeFrom = wordsIntFrom * wordSize + dig2bytes[leadingDigitsFrom];
if (digitsInt < digitsIntFrom) {
wordIdxFrom += (wordsIntFrom - wordsInt);
if (leadingDigitsFrom > 0) {
wordIdxFrom++;
}
if (leadingDigits > 0) {
wordIdxFrom--;
}
wordsIntFrom = wordsInt;
leadingDigitsFrom = leadingDigits;
// TODO overflow here
} else if (intSize > iSizeFrom) {
for (; intSize > iSizeFrom; ) {
intSize--;
bin[binIdx] = mask & 0xff;
binIdx++;
}
}
// when fracSize smaller than fracSizeFrom, output is truncated
if (fracSize < fracSizeFrom) {
wordsFracFrom = wordsFrac;
trailingDigitsFrom = trailingDigits;
// TODO truncated
} else if (fracSize > fracSizeFrom && trailingDigitsFrom > 0) {
if (wordsFrac == wordsFracFrom) {
trailingDigitsFrom = trailingDigits;
fracSize = fracSizeFrom;
} else {
wordsFracFrom++;
trailingDigitsFrom = 0;
}
}
// xIntFrom part
if (leadingDigitsFrom > 0) {
int i = dig2bytes[leadingDigitsFrom];
int x = (this.wordBuf[wordIdxFrom] % powers10[leadingDigitsFrom]) ^ mask;
wordIdxFrom++;
writeWord(bin, x, i, binIdx);
binIdx += i;
}
// wordsInt + wordsFrac part.
for (int stop = wordIdxFrom + wordsIntFrom + wordsFracFrom;
wordIdxFrom < stop;
binIdx += wordSize) {
int x = this.wordBuf[wordIdxFrom] ^ mask;
wordIdxFrom++;
writeWord(bin, x, 4, binIdx);
}
if (trailingDigitsFrom > 0) {
int x;
int i = dig2bytes[trailingDigitsFrom];
int lim = trailingDigits;
if (wordsFracFrom < wordsFrac) {
lim = digitsPerWord;
}
for (; trailingDigitsFrom < lim && dig2bytes[trailingDigitsFrom] == i; ) {
trailingDigitsFrom++;
}
x = (this.wordBuf[wordIdxFrom] / powers10[digitsPerWord - trailingDigitsFrom]) ^ mask;
writeWord(bin, x, i, binIdx);
binIdx += i;
}
if (fracSize > fracSizeFrom) {
int binIdxEnd = originIntSize + originFracSize;
for (; fracSize > fracSizeFrom && binIdx < binIdxEnd; ) {
fracSize--;
bin[binIdx] = mask & 0xff;
binIdx++;
}
}
bin[0] ^= 0x80;
return bin;
}
// write a word into buf.
private void writeWord(int[] b, int word, int size, int start) {
switch (size) {
case 1:
b[start] = word & 0xFF;
break;
case 2:
b[start] = (word >>> 8) & 0xFF;
b[start + 1] = word & 0xFF;
break;
case 3:
b[start] = (word >>> 16) & 0xFF;
b[start + 1] = (word >>> 8) & 0xFF;
b[start + 2] = word & 0xFF;
break;
case 4:
b[start] = (word >>> 24) & 0xFF;
b[start + 1] = (word >>> 16) & 0xFF;
b[start + 2] = (word >>> 8) & 0xFF;
b[start + 3] = word & 0xFF;
break;
}
}
/** Clears this instance. */
public void clear() {
this.digitsFrac = 0;
this.digitsInt = 0;
this.negative = false;
}
}

101
src/main/java/com/pingcap/tikv/event/CacheInvalidateEvent.java Normal file
View File

@ -0,0 +1,101 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.event;
import java.io.Serializable;
public class CacheInvalidateEvent implements Serializable {
public enum CacheType implements Serializable {
REGION_STORE,
REQ_FAILED,
LEADER
}
private long regionId;
private long storeId;
private boolean invalidateRegion;
private boolean invalidateStore;
private CacheType cacheType;
public CacheInvalidateEvent(
long regionId, long storeId, boolean updateRegion, boolean updateStore, CacheType type) {
this.regionId = regionId;
this.storeId = storeId;
this.cacheType = type;
if (updateRegion) {
invalidateRegion();
}
if (updateStore) {
invalidateStore();
}
}
public long getRegionId() {
return regionId;
}
public long getStoreId() {
return storeId;
}
@Override
public boolean equals(Object obj) {
if (obj == this) {
return true;
} else if (obj instanceof CacheInvalidateEvent) {
CacheInvalidateEvent event = (CacheInvalidateEvent) obj;
return event.getRegionId() == getRegionId()
&& event.getStoreId() == getStoreId()
&& event.getCacheType() == getCacheType();
}
return false;
}
@Override
public int hashCode() {
int result = 1106;
result += result * 31 + getStoreId();
result += result * 31 + getRegionId();
result += result * 31 + getCacheType().name().hashCode();
return result;
}
public void invalidateRegion() {
invalidateRegion = true;
}
public void invalidateStore() {
invalidateStore = true;
}
public boolean shouldUpdateRegion() {
return invalidateRegion;
}
public boolean shouldUpdateStore() {
return invalidateStore;
}
public CacheType getCacheType() {
return cacheType;
}
@Override
public String toString() {
return String.format("RegionId=%d,StoreId=%d,Type=%s", regionId, storeId, cacheType.name());
}
}

26
src/main/java/com/pingcap/tikv/exception/CastingException.java Normal file
View File

@ -0,0 +1,26 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.exception;
public class CastingException extends RuntimeException {
public CastingException(Exception e) {
super(e);
}
public CastingException(String msg) {
super(msg);
}
}

22
src/main/java/com/pingcap/tikv/exception/DAGRequestException.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.exception;
public class DAGRequestException extends RuntimeException {
public DAGRequestException(String msg) {
super(msg);
}
}

30
src/main/java/com/pingcap/tikv/exception/GrpcException.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.exception;
public class GrpcException extends RuntimeException {
public GrpcException(Exception e) {
super(e);
}
public GrpcException(String msg) {
super(msg);
}
public GrpcException(String msg, Exception e) {
super(msg, e);
}
}

22
src/main/java/com/pingcap/tikv/exception/IgnoreUnsupportedTypeException.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.exception;
public class IgnoreUnsupportedTypeException extends RuntimeException {
public IgnoreUnsupportedTypeException(String msg) {
super(msg);
}
}

22
src/main/java/com/pingcap/tikv/exception/InvalidCodecFormatException.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.exception;
public class InvalidCodecFormatException extends RuntimeException {
public InvalidCodecFormatException(String msg) {
super(msg);
}
}

35
src/main/java/com/pingcap/tikv/exception/KeyException.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.exception;
import com.pingcap.tikv.kvproto.Kvrpcpb;
public class KeyException extends RuntimeException {
private final Kvrpcpb.KeyError keyErr;
public KeyException(String errMsg) {
super(errMsg);
keyErr = null;
}
public KeyException(Kvrpcpb.KeyError keyErr) {
this.keyErr = keyErr;
}
public Kvrpcpb.KeyError getKeyErr() {
return keyErr;
}
}

30
src/main/java/com/pingcap/tikv/exception/RegionException.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.exception;
import com.pingcap.tikv.kvproto.Errorpb.Error;
public class RegionException extends RuntimeException {
private final Error regionErr;
public RegionException(Error regionErr) {
this.regionErr = regionErr;
}
public Error getRegionErr() {
return regionErr;
}
}

22
src/main/java/com/pingcap/tikv/exception/RegionTaskException.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright 2018 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.exception;
public class RegionTaskException extends RuntimeException {
public RegionTaskException(String msg, Throwable throwable) {
super(msg, throwable);
}
}

37
src/main/java/com/pingcap/tikv/exception/SelectException.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.exception;
import com.pingcap.tidb.tipb.Error;
public class SelectException extends RuntimeException {
private final Error err;
public SelectException(Error err, String msg) {
super(msg);
this.err = err;
}
// TODO: improve this
public SelectException(String msg) {
super(msg);
this.err = null;
}
public Error getError() {
return err;
}
}

26
src/main/java/com/pingcap/tikv/exception/TiClientInternalException.java Normal file
View File

@ -0,0 +1,26 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.exception;
public class TiClientInternalException extends RuntimeException {
public TiClientInternalException(String msg) {
super(msg);
}
public TiClientInternalException(String msg, Throwable t) {
super(msg, t);
}
}

26
src/main/java/com/pingcap/tikv/exception/TiExpressionException.java Normal file
View File

@ -0,0 +1,26 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.exception;
public class TiExpressionException extends RuntimeException {
public TiExpressionException(String msg) {
super(msg);
}
public TiExpressionException(String msg, Throwable t) {
super(msg, t);
}
}

26
src/main/java/com/pingcap/tikv/exception/TypeException.java Normal file
View File

@ -0,0 +1,26 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.exception;
public class TypeException extends RuntimeException {
public TypeException(String msg) {
super(msg);
}
public TypeException(String msg, Throwable t) {
super(msg, t);
}
}

22
src/main/java/com/pingcap/tikv/exception/UnsupportedTypeException.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.exception;
public class UnsupportedTypeException extends RuntimeException {
public UnsupportedTypeException(String msg) {
super(msg);
}
}

87
src/main/java/com/pingcap/tikv/expression/AggregateFunction.java Normal file
View File

@ -0,0 +1,87 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression;
import static java.util.Objects.requireNonNull;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Objects;
public class AggregateFunction implements Expression {
public enum FunctionType {
Sum,
Count,
Min,
Max,
First
}
private final FunctionType type;
private final Expression argument;
public static AggregateFunction newCall(FunctionType type, Expression argument) {
return new AggregateFunction(type, argument);
}
private AggregateFunction(FunctionType type, Expression argument) {
this.type = requireNonNull(type, "function type is null");
this.argument = requireNonNull(argument, "function argument is null");
}
public FunctionType getType() {
return type;
}
public Expression getArgument() {
return argument;
}
@Override
public List<Expression> getChildren() {
return ImmutableList.of(argument);
}
@Override
public <R, C> R accept(Visitor<R, C> visitor, C context) {
return visitor.visit(this, context);
}
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
if (!(other instanceof AggregateFunction)) {
return false;
}
AggregateFunction that = (AggregateFunction) other;
return type == that.type && Objects.equals(argument, that.argument);
}
@Override
public int hashCode() {
return Objects.hash(type, argument);
}
@Override
public String toString() {
return String.format(
"%s(%s)", getType(), Joiner.on(",").useForNull("NULL").join(getChildren()));
}
}

120
src/main/java/com/pingcap/tikv/expression/ArithmeticBinaryExpression.java Normal file
View File

@ -0,0 +1,120 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression;
import static com.pingcap.tikv.expression.ArithmeticBinaryExpression.Type.*;
import static java.util.Objects.requireNonNull;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Objects;
public class ArithmeticBinaryExpression implements Expression {
public enum Type {
PLUS,
MINUS,
MULTIPLY,
DIVIDE,
BIT_AND,
BIT_OR,
BIT_XOR
}
public static ArithmeticBinaryExpression plus(Expression left, Expression right) {
return new ArithmeticBinaryExpression(PLUS, left, right);
}
public static ArithmeticBinaryExpression minus(Expression left, Expression right) {
return new ArithmeticBinaryExpression(MINUS, left, right);
}
public static ArithmeticBinaryExpression multiply(Expression left, Expression right) {
return new ArithmeticBinaryExpression(MULTIPLY, left, right);
}
public static ArithmeticBinaryExpression divide(Expression left, Expression right) {
return new ArithmeticBinaryExpression(DIVIDE, left, right);
}
public static ArithmeticBinaryExpression bitAnd(Expression left, Expression right) {
return new ArithmeticBinaryExpression(BIT_AND, left, right);
}
public static ArithmeticBinaryExpression bitOr(Expression left, Expression right) {
return new ArithmeticBinaryExpression(BIT_OR, left, right);
}
public static ArithmeticBinaryExpression bitXor(Expression left, Expression right) {
return new ArithmeticBinaryExpression(BIT_XOR, left, right);
}
private final Expression left;
private final Expression right;
private final Type compType;
public ArithmeticBinaryExpression(Type type, Expression left, Expression right) {
this.left = requireNonNull(left, "left expression is null");
this.right = requireNonNull(right, "right expression is null");
this.compType = requireNonNull(type, "type is null");
}
public Expression getLeft() {
return left;
}
public Expression getRight() {
return right;
}
public Type getCompType() {
return compType;
}
@Override
public List<Expression> getChildren() {
return ImmutableList.of(left, right);
}
@Override
public <R, C> R accept(Visitor<R, C> visitor, C context) {
return visitor.visit(this, context);
}
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
if (!(other instanceof ArithmeticBinaryExpression)) {
return false;
}
ArithmeticBinaryExpression that = (ArithmeticBinaryExpression) other;
return (compType == that.compType)
&& Objects.equals(left, that.left)
&& Objects.equals(right, that.right);
}
@Override
public int hashCode() {
return Objects.hash(compType, left, right);
}
@Override
public String toString() {
return String.format("[%s %s %s]", getLeft(), getCompType(), getRight());
}
}

53
src/main/java/com/pingcap/tikv/expression/Blacklist.java Normal file
View File

@ -0,0 +1,53 @@
/*
*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.pingcap.tikv.expression;
import static java.util.Objects.requireNonNull;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
public class Blacklist {
private final Set<String> unsupported = new HashSet<>();
Blacklist(String string) {
if (string != null) {
String[] some = string.split(",");
for (String one : some) {
String trimmedExprName = one.trim();
if (!trimmedExprName.isEmpty()) {
unsupported.add(one.trim());
}
}
}
}
boolean isUnsupported(String name) {
return unsupported.contains(name);
}
boolean isUnsupported(Class<?> cls) {
return isUnsupported(requireNonNull(cls).getSimpleName());
}
@Override
public String toString() {
return unsupported.stream().collect(Collectors.joining(","));
}
}

55
src/main/java/com/pingcap/tikv/expression/ByItem.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression;
import static com.google.common.base.Preconditions.checkNotNull;
import com.pingcap.tikv.expression.visitor.ProtoConverter;
import java.io.Serializable;
public class ByItem implements Serializable {
private Expression expr;
private boolean desc;
public static ByItem create(Expression expr, boolean desc) {
return new ByItem(expr, desc);
}
private ByItem(Expression expr, boolean desc) {
checkNotNull(expr, "Expr cannot be null for ByItem");
this.expr = expr;
this.desc = desc;
}
public com.pingcap.tidb.tipb.ByItem toProto(Object context) {
com.pingcap.tidb.tipb.ByItem.Builder builder = com.pingcap.tidb.tipb.ByItem.newBuilder();
return builder.setExpr(ProtoConverter.toProto(expr, context)).setDesc(desc).build();
}
public Expression getExpr() {
return expr;
}
public boolean isDesc() {
return desc;
}
@Override
public String toString() {
return String.format("[%s %s]", expr.toString(), desc ? "DESC" : "ASC");
}
}

143
src/main/java/com/pingcap/tikv/expression/ColumnRef.java Normal file
View File

@ -0,0 +1,143 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression;
import com.google.common.collect.ImmutableList;
import com.pingcap.tikv.exception.TiClientInternalException;
import com.pingcap.tikv.exception.TiExpressionException;
import com.pingcap.tikv.meta.TiColumnInfo;
import com.pingcap.tikv.meta.TiTableInfo;
import com.pingcap.tikv.types.DataType;
import java.util.List;
import java.util.Objects;
public class ColumnRef implements Expression {
public static ColumnRef create(String name, TiTableInfo table) {
for (TiColumnInfo columnInfo : table.getColumns()) {
if (columnInfo.matchName(name)) {
return new ColumnRef(columnInfo.getName(), columnInfo, table);
}
}
throw new TiExpressionException(
String.format("Column name %s not found in table %s", name, table));
}
public static ColumnRef create(String name) {
return new ColumnRef(name);
}
private final String name;
private TiColumnInfo columnInfo;
private TiTableInfo tableInfo;
public ColumnRef(String name) {
this.name = name;
}
public ColumnRef(String name, TiColumnInfo columnInfo, TiTableInfo tableInfo) {
this.name = name;
this.columnInfo = columnInfo;
this.tableInfo = tableInfo;
}
public String getName() {
return name;
}
public void resolve(TiTableInfo table) {
TiColumnInfo columnInfo = null;
for (TiColumnInfo col : table.getColumns()) {
if (col.matchName(name)) {
columnInfo = col;
break;
}
}
if (columnInfo == null) {
throw new TiExpressionException(
String.format("No Matching column %s from table %s", name, table.getName()));
}
if (columnInfo.getId() == 0) {
throw new TiExpressionException("Zero Id is not a referable column id");
}
this.tableInfo = table;
this.columnInfo = columnInfo;
}
public TiColumnInfo getColumnInfo() {
if (columnInfo == null) {
throw new TiClientInternalException(String.format("ColumnRef [%s] is unbound", name));
}
return columnInfo;
}
public DataType getType() {
return getColumnInfo().getType();
}
public TiTableInfo getTableInfo() {
return tableInfo;
}
public boolean isResolved() {
return tableInfo != null && columnInfo != null;
}
@Override
public boolean equals(Object another) {
if (this == another) {
return true;
}
if (another instanceof ColumnRef) {
ColumnRef that = (ColumnRef) another;
if (isResolved() && that.isResolved()) {
return Objects.equals(columnInfo, that.columnInfo)
&& Objects.equals(tableInfo, that.tableInfo);
} else {
return name.equalsIgnoreCase(that.name);
}
} else {
return false;
}
}
@Override
public int hashCode() {
if (isResolved()) {
return Objects.hash(tableInfo, columnInfo);
} else {
return Objects.hashCode(name);
}
}
@Override
public String toString() {
return String.format("[%s]", getName());
}
@Override
public List<Expression> getChildren() {
return ImmutableList.of();
}
@Override
public <R, C> R accept(Visitor<R, C> visitor, C context) {
return visitor.visit(this, context);
}
}

199
src/main/java/com/pingcap/tikv/expression/ComparisonBinaryExpression.java Normal file
View File

@ -0,0 +1,199 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression;
import static com.google.common.base.Preconditions.checkArgument;
import static com.pingcap.tikv.expression.ComparisonBinaryExpression.Type.*;
import static java.util.Objects.requireNonNull;
import com.google.common.collect.ImmutableList;
import com.pingcap.tikv.exception.TiExpressionException;
import com.pingcap.tikv.key.TypedKey;
import com.pingcap.tikv.types.DataType;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
public class ComparisonBinaryExpression implements Expression {
public enum Type {
EQUAL,
NOT_EQUAL,
LESS_THAN,
LESS_EQUAL,
GREATER_THAN,
GREATER_EQUAL
}
public static ComparisonBinaryExpression equal(Expression left, Expression right) {
return new ComparisonBinaryExpression(EQUAL, left, right);
}
public static ComparisonBinaryExpression notEqual(Expression left, Expression right) {
return new ComparisonBinaryExpression(NOT_EQUAL, left, right);
}
public static ComparisonBinaryExpression lessThan(Expression left, Expression right) {
return new ComparisonBinaryExpression(LESS_THAN, left, right);
}
public static ComparisonBinaryExpression lessEqual(Expression left, Expression right) {
return new ComparisonBinaryExpression(LESS_EQUAL, left, right);
}
public static ComparisonBinaryExpression greaterThan(Expression left, Expression right) {
return new ComparisonBinaryExpression(GREATER_THAN, left, right);
}
public static ComparisonBinaryExpression greaterEqual(Expression left, Expression right) {
return new ComparisonBinaryExpression(GREATER_EQUAL, left, right);
}
public static class NormalizedPredicate {
private final ComparisonBinaryExpression pred;
private TypedKey key;
NormalizedPredicate(ComparisonBinaryExpression pred) {
checkArgument(pred.getLeft() instanceof ColumnRef);
checkArgument(pred.getRight() instanceof Constant);
this.pred = pred;
}
public ColumnRef getColumnRef() {
return (ColumnRef) pred.getLeft();
}
public Constant getValue() {
return (Constant) pred.getRight();
}
public Type getType() {
return pred.getComparisonType();
}
public TypedKey getTypedLiteral() {
return getTypedLiteral(DataType.UNSPECIFIED_LEN);
}
public TypedKey getTypedLiteral(int prefixLength) {
if (key == null) {
key = TypedKey.toTypedKey(getValue().getValue(), getColumnRef().getType(), prefixLength);
}
return key;
}
}
private final Expression left;
private final Expression right;
private final Type compType;
private transient Optional<NormalizedPredicate> normalizedPredicate;
public ComparisonBinaryExpression(Type type, Expression left, Expression right) {
this.left = requireNonNull(left, "left expression is null");
this.right = requireNonNull(right, "right expression is null");
this.compType = requireNonNull(type, "type is null");
}
@Override
public List<Expression> getChildren() {
return ImmutableList.of(left, right);
}
@Override
public <R, C> R accept(Visitor<R, C> visitor, C context) {
return visitor.visit(this, context);
}
public Expression getLeft() {
return left;
}
public Expression getRight() {
return right;
}
public Type getComparisonType() {
return compType;
}
public NormalizedPredicate normalize() {
if (normalizedPredicate != null) {
return normalizedPredicate.orElseGet(null);
}
if (getLeft() instanceof Constant && getRight() instanceof ColumnRef) {
Constant left = (Constant) getLeft();
ColumnRef right = (ColumnRef) getRight();
Type newType;
switch (getComparisonType()) {
case EQUAL:
newType = EQUAL;
break;
case LESS_EQUAL:
newType = GREATER_EQUAL;
break;
case LESS_THAN:
newType = GREATER_THAN;
break;
case GREATER_EQUAL:
newType = LESS_EQUAL;
break;
case GREATER_THAN:
newType = LESS_THAN;
break;
case NOT_EQUAL:
newType = NOT_EQUAL;
break;
default:
throw new TiExpressionException(
String.format(
"PredicateNormalizer is not able to process type %s", getComparisonType()));
}
ComparisonBinaryExpression newExpression =
new ComparisonBinaryExpression(newType, right, left);
normalizedPredicate = Optional.of(new NormalizedPredicate(newExpression));
return normalizedPredicate.get();
} else if (getRight() instanceof Constant && getLeft() instanceof ColumnRef) {
normalizedPredicate = Optional.of(new NormalizedPredicate(this));
return normalizedPredicate.get();
} else {
return null;
}
}
@Override
public String toString() {
return String.format("[%s %s %s]", getLeft(), getComparisonType(), getRight());
}
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
if (!(other instanceof ComparisonBinaryExpression)) {
return false;
}
ComparisonBinaryExpression that = (ComparisonBinaryExpression) other;
return (compType == that.compType)
&& Objects.equals(left, that.left)
&& Objects.equals(right, that.right);
}
@Override
public int hashCode() {
return Objects.hash(compType, left, right);
}
}

127
src/main/java/com/pingcap/tikv/expression/Constant.java Normal file
View File

@ -0,0 +1,127 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression;
import com.google.common.collect.ImmutableList;
import com.pingcap.tikv.exception.TiExpressionException;
import com.pingcap.tikv.types.*;
import java.math.BigDecimal;
import java.sql.Date;
import java.sql.Timestamp;
import java.util.List;
import java.util.Objects;
import org.joda.time.DateTime;
// Refactor needed.
// Refer to https://github.com/pingcap/tipb/blob/master/go-tipb/expression.pb.go
// TODO: This might need a refactor to accept an DataType?
public class Constant implements Expression {
private final Object value;
private DataType type;
public static Constant create(Object value, DataType type) {
return new Constant(value, type);
}
public static Constant create(Object value) {
return new Constant(value, null);
}
public Constant(Object value, DataType type) {
this.value = value;
this.type = (type == null && value != null) ? getDefaultType(value) : type;
}
protected static boolean isIntegerType(Object value) {
return value instanceof Long
|| value instanceof Integer
|| value instanceof Short
|| value instanceof Byte;
}
private static DataType getDefaultType(Object value) {
if (value == null) {
throw new TiExpressionException("NULL constant has no type");
} else if (isIntegerType(value)) {
return IntegerType.BIGINT;
} else if (value instanceof String) {
return StringType.VARCHAR;
} else if (value instanceof Float) {
return RealType.FLOAT;
} else if (value instanceof Double) {
return RealType.DOUBLE;
} else if (value instanceof BigDecimal) {
return DecimalType.DECIMAL;
} else if (value instanceof DateTime) {
return DateTimeType.DATETIME;
} else if (value instanceof Date) {
return DateType.DATE;
} else if (value instanceof Timestamp) {
return TimestampType.TIMESTAMP;
} else if (value instanceof byte[]) {
return BytesType.TEXT;
} else {
throw new TiExpressionException(
"Constant type not supported:" + value.getClass().getSimpleName());
}
}
public void setType(DataType type) {
this.type = type;
}
public Object getValue() {
return value;
}
public DataType getType() {
return type;
}
@Override
public String toString() {
if (value == null) {
return "null";
}
if (value instanceof String) {
return String.format("\"%s\"", value);
}
return value.toString();
}
@Override
public boolean equals(Object other) {
if (other instanceof Constant) {
return Objects.equals(value, ((Constant) other).value);
}
return false;
}
@Override
public int hashCode() {
return Objects.hashCode(value);
}
@Override
public List<Expression> getChildren() {
return ImmutableList.of();
}
@Override
public <R, C> R accept(Visitor<R, C> visitor, C context) {
return visitor.visit(this, context);
}
}

25
src/main/java/com/pingcap/tikv/expression/Expression.java Normal file
View File

@ -0,0 +1,25 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression;
import java.io.Serializable;
import java.util.List;
public interface Expression extends Serializable {
List<Expression> getChildren();
<R, C> R accept(Visitor<R, C> visitor, C context);
}

27
src/main/java/com/pingcap/tikv/expression/ExpressionBlacklist.java Normal file
View File

@ -0,0 +1,27 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression;
public class ExpressionBlacklist extends Blacklist {
public ExpressionBlacklist(String exprsString) {
super(exprsString);
}
public boolean isUnsupportedPushdownExpr(Class<?> cls) {
return isUnsupported(cls);
}
}

67
src/main/java/com/pingcap/tikv/expression/IsNull.java Normal file
View File

@ -0,0 +1,67 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression;
import static java.util.Objects.requireNonNull;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Objects;
public class IsNull implements Expression {
private Expression expression;
public IsNull(Expression expression) {
this.expression = requireNonNull(expression, "expression is null");
}
public Expression getExpression() {
return expression;
}
@Override
public List<Expression> getChildren() {
return ImmutableList.of(expression);
}
@Override
public <R, C> R accept(Visitor<R, C> visitor, C context) {
return visitor.visit(this, context);
}
@Override
public String toString() {
return String.format("IsNull(%s)", getExpression());
}
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
if (!(other instanceof IsNull)) {
return false;
}
IsNull that = (IsNull) other;
return Objects.equals(expression, that.expression);
}
@Override
public int hashCode() {
return Objects.hashCode(expression);
}
}

87
src/main/java/com/pingcap/tikv/expression/LogicalBinaryExpression.java Normal file
View File

@ -0,0 +1,87 @@
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: expression.proto
package com.pingcap.tikv.expression;
import static java.util.Objects.requireNonNull;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Objects;
public class LogicalBinaryExpression implements Expression {
public enum Type {
AND,
OR,
XOR
}
public static LogicalBinaryExpression and(Expression left, Expression right) {
return new LogicalBinaryExpression(Type.AND, left, right);
}
public static LogicalBinaryExpression or(Expression left, Expression right) {
return new LogicalBinaryExpression(Type.OR, left, right);
}
public static LogicalBinaryExpression xor(Expression left, Expression right) {
return new LogicalBinaryExpression(Type.XOR, left, right);
}
public LogicalBinaryExpression(Type type, Expression left, Expression right) {
this.left = requireNonNull(left, "left expression is null");
this.right = requireNonNull(right, "right expression is null");
this.compType = requireNonNull(type, "type is null");
}
@Override
public List<Expression> getChildren() {
return ImmutableList.of(getLeft(), getRight());
}
@Override
public <R, C> R accept(Visitor<R, C> visitor, C context) {
return visitor.visit(this, context);
}
public Expression getLeft() {
return left;
}
public Expression getRight() {
return right;
}
public Type getCompType() {
return compType;
}
private final Expression left;
private final Expression right;
private final Type compType;
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
if (!(other instanceof LogicalBinaryExpression)) {
return false;
}
LogicalBinaryExpression that = (LogicalBinaryExpression) other;
return (compType == that.compType)
&& Objects.equals(left, that.left)
&& Objects.equals(right, that.right);
}
@Override
public int hashCode() {
return Objects.hash(compType, left, right);
}
@Override
public String toString() {
return String.format("[%s %s %s]", getLeft(), getCompType(), getRight());
}
}

72
src/main/java/com/pingcap/tikv/expression/Not.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression;
import static java.util.Objects.requireNonNull;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Objects;
public class Not implements Expression {
public static Not not(Expression expression) {
return new Not(expression);
}
private Expression expression;
public Not(Expression expression) {
this.expression = requireNonNull(expression, "expression is null");
}
public Expression getExpression() {
return expression;
}
@Override
public List<Expression> getChildren() {
return ImmutableList.of(expression);
}
@Override
public <R, C> R accept(Visitor<R, C> visitor, C context) {
return visitor.visit(this, context);
}
@Override
public String toString() {
return String.format("Not(%s)", getExpression());
}
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
if (!(other instanceof Not)) {
return false;
}
Not that = (Not) other;
return Objects.equals(expression, that.expression);
}
@Override
public int hashCode() {
return Objects.hashCode(expression);
}
}

145
src/main/java/com/pingcap/tikv/expression/StringRegExpression.java Normal file
View File

@ -0,0 +1,145 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression;
import static com.pingcap.tikv.expression.StringRegExpression.Type.*;
import static java.util.Objects.requireNonNull;
import com.google.common.collect.ImmutableList;
import com.pingcap.tikv.key.TypedKey;
import com.pingcap.tikv.types.DataType;
import com.pingcap.tikv.types.IntegerType;
import java.util.List;
import java.util.Objects;
public class StringRegExpression implements Expression {
public enum Type {
STARTS_WITH,
CONTAINS,
ENDS_WITH,
LIKE
}
public static StringRegExpression startsWith(Expression left, Expression right) {
Expression reg =
Constant.create(((Constant) right).getValue() + "%", ((Constant) right).getType());
return new StringRegExpression(STARTS_WITH, left, right, reg);
}
public static StringRegExpression contains(Expression left, Expression right) {
Expression reg =
Constant.create("%" + ((Constant) right).getValue() + "%", ((Constant) right).getType());
return new StringRegExpression(CONTAINS, left, right, reg);
}
public static StringRegExpression endsWith(Expression left, Expression right) {
Expression reg =
Constant.create("%" + ((Constant) right).getValue(), ((Constant) right).getType());
return new StringRegExpression(ENDS_WITH, left, right, reg);
}
public static StringRegExpression like(Expression left, Expression right) {
return new StringRegExpression(LIKE, left, right, right);
}
private transient TypedKey key;
public ColumnRef getColumnRef() {
return (ColumnRef) getLeft();
}
public Constant getValue() {
return (Constant) getRight();
}
public TypedKey getTypedLiteral() {
return getTypedLiteral(DataType.UNSPECIFIED_LEN);
}
public TypedKey getTypedLiteral(int prefixLength) {
if (key == null) {
key = TypedKey.toTypedKey(getValue().getValue(), getColumnRef().getType(), prefixLength);
}
return key;
}
private final Expression left;
private final Expression right;
private final Expression reg;
private final Type regType;
public StringRegExpression(Type type, Expression left, Expression right, Expression reg) {
this.left = requireNonNull(left, "left expression is null");
this.right = requireNonNull(right, "right expression is null");
this.regType = requireNonNull(type, "type is null");
this.reg = requireNonNull(reg, "reg string is null");
}
@Override
public List<Expression> getChildren() {
// For LIKE statement, an extra ESCAPE parameter is required as the third parameter for
// ScalarFunc.
// However in Spark ESCAPE is not supported so we simply set this value to zero.
return ImmutableList.of(left, reg, Constant.create(0, IntegerType.BIGINT));
}
@Override
public <R, C> R accept(Visitor<R, C> visitor, C context) {
return visitor.visit(this, context);
}
public Expression getLeft() {
return left;
}
public Expression getRight() {
return right;
}
public Type getRegType() {
return regType;
}
public Expression getReg() {
return reg;
}
@Override
public String toString() {
return String.format("[%s %s %s reg: %s]", getLeft(), getRegType(), getRight(), getReg());
}
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
if (!(other instanceof StringRegExpression)) {
return false;
}
StringRegExpression that = (StringRegExpression) other;
return (regType == that.regType)
&& Objects.equals(left, that.left)
&& Objects.equals(left, that.right)
&& Objects.equals(reg, that.reg);
}
@Override
public int hashCode() {
return Objects.hash(regType, left, right, reg);
}
}

67
src/main/java/com/pingcap/tikv/expression/TypeBlacklist.java Normal file
View File

@ -0,0 +1,67 @@
/*
*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.pingcap.tikv.expression;
import static com.pingcap.tikv.types.MySQLType.*;
import com.pingcap.tikv.types.MySQLType;
import java.util.HashMap;
import java.util.Map;
public class TypeBlacklist extends Blacklist {
private static final Map<MySQLType, String> typeToMySQLMap = initialTypeMap();
private static HashMap<MySQLType, String> initialTypeMap() {
HashMap<MySQLType, String> map = new HashMap<>();
map.put(TypeDecimal, "decimal");
map.put(TypeTiny, "tinyint");
map.put(TypeShort, "smallint");
map.put(TypeLong, "int");
map.put(TypeFloat, "float");
map.put(TypeDouble, "double");
map.put(TypeNull, "null");
map.put(TypeTimestamp, "timestamp");
map.put(TypeLonglong, "bigint");
map.put(TypeInt24, "mediumint");
map.put(TypeDate, "date");
map.put(TypeDuration, "time");
map.put(TypeDatetime, "datetime");
map.put(TypeYear, "year");
map.put(TypeNewDate, "date");
map.put(TypeVarchar, "varchar");
map.put(TypeJSON, "json");
map.put(TypeNewDecimal, "decimal");
map.put(TypeEnum, "enum");
map.put(TypeSet, "set");
map.put(TypeTinyBlob, "tinytext");
map.put(TypeMediumBlob, "mediumtext");
map.put(TypeLongBlob, "longtext");
map.put(TypeBlob, "text");
map.put(TypeVarString, "varString");
map.put(TypeString, "string");
return map;
}
public TypeBlacklist(String typesString) {
super(typesString);
}
public boolean isUnsupportedType(MySQLType sqlType) {
return isUnsupported(typeToMySQLMap.getOrDefault(sqlType, ""));
}
}

38
src/main/java/com/pingcap/tikv/expression/Visitor.java Normal file
View File

@ -0,0 +1,38 @@
/*
*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.pingcap.tikv.expression;
public abstract class Visitor<R, C> {
protected abstract R visit(ColumnRef node, C context);
protected abstract R visit(ComparisonBinaryExpression node, C context);
protected abstract R visit(StringRegExpression node, C context);
protected abstract R visit(ArithmeticBinaryExpression node, C context);
protected abstract R visit(LogicalBinaryExpression node, C context);
protected abstract R visit(Constant node, C context);
protected abstract R visit(AggregateFunction node, C context);
protected abstract R visit(IsNull node, C context);
protected abstract R visit(Not node, C context);
}

42
src/main/java/com/pingcap/tikv/expression/visitor/ColumnMatcher.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright 2018 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression.visitor;
import com.pingcap.tikv.expression.ColumnRef;
import com.pingcap.tikv.expression.Expression;
public class ColumnMatcher extends DefaultVisitor<Boolean, Void> {
private final ColumnRef columnRef;
private ColumnMatcher(ColumnRef exp) {
this.columnRef = exp;
}
public static Boolean match(ColumnRef col, Expression expression) {
ColumnMatcher matcher = new ColumnMatcher(col);
return expression.accept(matcher, null);
}
@Override
protected Boolean process(Expression node, Void context) {
return false;
}
@Override
protected Boolean visit(ColumnRef node, Void context) {
return node.getColumnInfo().matchName(columnRef.getName());
}
}

72
src/main/java/com/pingcap/tikv/expression/visitor/DefaultVisitor.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression.visitor;
import com.pingcap.tikv.expression.*;
public class DefaultVisitor<R, C> extends Visitor<R, C> {
protected R process(Expression node, C context) {
for (Expression expr : node.getChildren()) {
expr.accept(this, context);
}
return null;
}
@Override
protected R visit(ColumnRef node, C context) {
return process(node, context);
}
@Override
protected R visit(ComparisonBinaryExpression node, C context) {
return process(node, context);
}
@Override
protected R visit(StringRegExpression node, C context) {
return process(node, context);
}
@Override
protected R visit(ArithmeticBinaryExpression node, C context) {
return process(node, context);
}
@Override
protected R visit(LogicalBinaryExpression node, C context) {
return process(node, context);
}
@Override
protected R visit(Constant node, C context) {
return process(node, context);
}
@Override
protected R visit(AggregateFunction node, C context) {
return process(node, context);
}
@Override
protected R visit(IsNull node, C context) {
return process(node, context);
}
@Override
protected R visit(Not node, C context) {
return process(node, context);
}
}

224
src/main/java/com/pingcap/tikv/expression/visitor/ExpressionTypeCoercer.java Normal file
View File

@ -0,0 +1,224 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression.visitor;
import static java.util.Objects.requireNonNull;
import com.pingcap.tikv.exception.TiExpressionException;
import com.pingcap.tikv.expression.*;
import com.pingcap.tikv.expression.AggregateFunction.FunctionType;
import com.pingcap.tikv.types.DataType;
import com.pingcap.tikv.types.DecimalType;
import com.pingcap.tikv.types.IntegerType;
import com.pingcap.tikv.types.RealType;
import com.pingcap.tikv.util.Pair;
import java.util.IdentityHashMap;
import java.util.List;
/**
* Validate and infer expression type Collected results are returned getTypeMap For now we don't do
* any type promotion and only coerce from left to right.
*/
public class ExpressionTypeCoercer extends Visitor<Pair<DataType, Double>, DataType> {
private final IdentityHashMap<Expression, DataType> typeMap = new IdentityHashMap<>();
private static final double MAX_CREDIBILITY = 1.0;
private static final double MIN_CREDIBILITY = 0.1;
private static final double COLUMN_REF_CRED = MAX_CREDIBILITY;
private static final double CONSTANT_CRED = MIN_CREDIBILITY;
private static final double LOGICAL_OP_CRED = MAX_CREDIBILITY;
private static final double COMPARISON_OP_CRED = MAX_CREDIBILITY;
private static final double SRING_REG_OP_CRED = MAX_CREDIBILITY;
private static final double FUNCTION_CRED = MAX_CREDIBILITY;
private static final double ISNULL_CRED = MAX_CREDIBILITY;
private static final double NOT_CRED = MAX_CREDIBILITY;
public IdentityHashMap<Expression, DataType> getTypeMap() {
return typeMap;
}
public static DataType inferType(Expression expression) {
ExpressionTypeCoercer inf = new ExpressionTypeCoercer();
return inf.infer(expression);
}
public DataType infer(Expression expression) {
requireNonNull(expression, "expression is null");
return expression.accept(this, null).first;
}
public void infer(List<? extends Expression> expressions) {
requireNonNull(expressions, "expressions is null");
expressions.forEach(expr -> expr.accept(this, null));
}
@Override
protected Pair<DataType, Double> visit(ColumnRef node, DataType targetType) {
DataType type = node.getType();
if (targetType != null && !targetType.equals(type)) {
throw new TiExpressionException(String.format("Column %s cannot be %s", node, targetType));
}
typeMap.put(node, type);
return Pair.create(type, COLUMN_REF_CRED);
}
// Try to coerceType if needed
// A column reference is source of coerce and constant is the subject to coerce
// targetType null means no coerce needed from parent and choose the highest credibility result
protected Pair<DataType, Double> coerceType(DataType targetType, Expression... nodes) {
if (nodes.length == 0) {
throw new TiExpressionException("failed to verify empty node list");
}
if (targetType == null) {
Pair<DataType, Double> baseline = nodes[0].accept(this, null);
for (int i = 1; i < nodes.length; i++) {
Pair<DataType, Double> current = nodes[i].accept(this, null);
if (current.second > baseline.second) {
baseline = current;
}
}
for (Expression node : nodes) {
node.accept(this, baseline.first);
}
return baseline;
} else {
double credibility = -1;
for (Expression node : nodes) {
Pair<DataType, Double> result = node.accept(this, targetType);
if (result.second > credibility) {
credibility = result.second;
}
}
return Pair.create(targetType, credibility);
}
}
@Override
protected Pair<DataType, Double> visit(ComparisonBinaryExpression node, DataType targetType) {
if (targetType != null && !targetType.equals(IntegerType.BOOLEAN)) {
throw new TiExpressionException(String.format("Comparison result cannot be %s", targetType));
}
if (!typeMap.containsKey(node)) {
coerceType(null, node.getLeft(), node.getRight());
typeMap.put(node, IntegerType.BOOLEAN);
}
return Pair.create(IntegerType.BOOLEAN, COMPARISON_OP_CRED);
}
@Override
protected Pair<DataType, Double> visit(StringRegExpression node, DataType targetType) {
if (targetType != null && !targetType.equals(IntegerType.BOOLEAN)) {
throw new TiExpressionException(String.format("Comparison result cannot be %s", targetType));
}
if (!typeMap.containsKey(node)) {
coerceType(null, node.getLeft(), node.getRight());
typeMap.put(node, IntegerType.BOOLEAN);
}
return Pair.create(IntegerType.BOOLEAN, SRING_REG_OP_CRED);
}
@Override
protected Pair<DataType, Double> visit(ArithmeticBinaryExpression node, DataType targetType) {
Pair<DataType, Double> result = coerceType(targetType, node.getLeft(), node.getRight());
typeMap.put(node, result.first);
return result;
}
@Override
protected Pair<DataType, Double> visit(LogicalBinaryExpression node, DataType targetType) {
if (targetType != null && !targetType.equals(IntegerType.BOOLEAN)) {
throw new TiExpressionException(String.format("Comparison result cannot be %s", targetType));
}
if (!typeMap.containsKey(node)) {
coerceType(null, node.getLeft(), node.getRight());
typeMap.put(node, IntegerType.BOOLEAN);
}
return Pair.create(IntegerType.BOOLEAN, LOGICAL_OP_CRED);
}
@Override
protected Pair<DataType, Double> visit(Constant node, DataType targetType) {
if (targetType == null) {
return Pair.create(node.getType(), CONSTANT_CRED);
} else {
node.setType(targetType);
typeMap.put(node, targetType);
return Pair.create(targetType, CONSTANT_CRED);
}
}
@Override
protected Pair<DataType, Double> visit(AggregateFunction node, DataType targetType) {
FunctionType fType = node.getType();
coerceType(null, node.getArgument());
switch (fType) {
case Count:
{
if (targetType != null && targetType.equals(IntegerType.BIGINT)) {
throw new TiExpressionException(String.format("Count cannot be %s", targetType));
}
typeMap.put(node, IntegerType.BIGINT);
return Pair.create(targetType, FUNCTION_CRED);
}
case Sum:
{
if (targetType != null && targetType.equals(DecimalType.DECIMAL)) {
throw new TiExpressionException(String.format("Sum cannot be %s", targetType));
}
DataType colType = node.getArgument().accept(this, null).first;
if (colType instanceof RealType) {
typeMap.put(node, RealType.DOUBLE);
} else {
typeMap.put(node, DecimalType.DECIMAL);
}
return Pair.create(targetType, FUNCTION_CRED);
}
case First:
case Max:
case Min:
{
Pair<DataType, Double> result = coerceType(targetType, node.getArgument());
typeMap.put(node, result.first);
return result;
}
default:
throw new TiExpressionException(String.format("Unknown function %s", fType));
}
}
@Override
protected Pair<DataType, Double> visit(IsNull node, DataType targetType) {
if (targetType != null && !targetType.equals(IntegerType.BOOLEAN)) {
throw new TiExpressionException(String.format("IsNull result cannot be %s", targetType));
}
if (!typeMap.containsKey(node)) {
coerceType(null, node.getExpression());
typeMap.put(node, IntegerType.BOOLEAN);
}
return Pair.create(IntegerType.BOOLEAN, ISNULL_CRED);
}
@Override
protected Pair<DataType, Double> visit(Not node, DataType targetType) {
if (targetType != null && !targetType.equals(IntegerType.BOOLEAN)) {
throw new TiExpressionException(String.format("Not result cannot be %s", targetType));
}
if (!typeMap.containsKey(node)) {
coerceType(null, node.getExpression());
typeMap.put(node, IntegerType.BOOLEAN);
}
return Pair.create(IntegerType.BOOLEAN, NOT_CRED);
}
}

111
src/main/java/com/pingcap/tikv/expression/visitor/IndexMatcher.java Normal file
View File

@ -0,0 +1,111 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression.visitor;
import static java.util.Objects.requireNonNull;
import com.pingcap.tikv.expression.*;
import com.pingcap.tikv.expression.ComparisonBinaryExpression.NormalizedPredicate;
import com.pingcap.tikv.meta.TiIndexColumn;
/**
* Test if a predicate matches and index column entirely and can be convert to index related ranges
* If a predicate matches only partially, it returns false
*/
public class IndexMatcher extends DefaultVisitor<Boolean, Void> {
private final boolean matchEqualTestOnly;
private final TiIndexColumn indexColumn;
private IndexMatcher(TiIndexColumn indexColumn, boolean matchEqualTestOnly) {
this.matchEqualTestOnly = matchEqualTestOnly;
this.indexColumn = requireNonNull(indexColumn, "index column is null");
}
public static IndexMatcher equalOnlyMatcher(TiIndexColumn indexColumn) {
return new IndexMatcher(indexColumn, true);
}
public static IndexMatcher matcher(TiIndexColumn indexColumn) {
return new IndexMatcher(indexColumn, false);
}
public boolean match(Expression expression) {
return expression.accept(this, null);
}
@Override
protected Boolean process(Expression node, Void context) {
return false;
}
@Override
protected Boolean visit(ColumnRef node, Void context) {
String indexColumnName = indexColumn.getName();
return node.getColumnInfo().matchName(indexColumnName);
}
@Override
protected Boolean visit(ComparisonBinaryExpression node, Void context) {
switch (node.getComparisonType()) {
case LESS_THAN:
case LESS_EQUAL:
case GREATER_THAN:
case GREATER_EQUAL:
case NOT_EQUAL:
if (matchEqualTestOnly) {
return false;
}
case EQUAL:
NormalizedPredicate predicate = node.normalize();
if (predicate == null) {
return false;
}
return predicate.getColumnRef().accept(this, context);
default:
return false;
}
}
@Override
protected Boolean visit(StringRegExpression node, Void context) {
switch (node.getRegType()) {
// If the predicate is StartsWith(col, 'a'), this predicate
// indicates a range of ['a', +∞) which can be used by index scan
case STARTS_WITH:
if (matchEqualTestOnly) {
return false;
}
return node.getLeft().accept(this, context);
default:
return false;
}
}
@Override
protected Boolean visit(LogicalBinaryExpression node, Void context) {
switch (node.getCompType()) {
case AND:
if (matchEqualTestOnly) {
return false;
}
case OR:
case XOR:
return node.getLeft().accept(this, context) && node.getRight().accept(this, context);
default:
return false;
}
}
}

200
src/main/java/com/pingcap/tikv/expression/visitor/IndexRangeBuilder.java Normal file
View File

@ -0,0 +1,200 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression.visitor;
import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
import com.google.common.collect.TreeRangeSet;
import com.pingcap.tikv.exception.TiExpressionException;
import com.pingcap.tikv.expression.*;
import com.pingcap.tikv.expression.ComparisonBinaryExpression.NormalizedPredicate;
import com.pingcap.tikv.key.TypedKey;
import com.pingcap.tikv.meta.TiIndexColumn;
import com.pingcap.tikv.meta.TiIndexInfo;
import com.pingcap.tikv.meta.TiTableInfo;
import com.pingcap.tikv.types.DataType;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
public class IndexRangeBuilder extends DefaultVisitor<RangeSet<TypedKey>, Void> {
private final Map<ColumnRef, Integer> lengths; // length of corresponding ColumnRef
public IndexRangeBuilder(TiTableInfo table, TiIndexInfo index) {
Map<ColumnRef, Integer> result = new HashMap<>();
if (table != null && index != null) {
for (TiIndexColumn indexColumn : index.getIndexColumns()) {
ColumnRef columnRef = ColumnRef.create(indexColumn.getName(), table);
result.put(columnRef, (int) indexColumn.getLength());
}
}
this.lengths = result;
}
public Set<Range<TypedKey>> buildRange(Expression predicate) {
Objects.requireNonNull(predicate, "predicate is null");
return predicate.accept(this, null).asRanges();
}
private static void throwOnError(Expression node) {
final String errorFormat = "Unsupported conversion to Range: %s";
throw new TiExpressionException(String.format(errorFormat, node));
}
protected RangeSet<TypedKey> process(Expression node, Void context) {
throwOnError(node);
return null;
}
@Override
protected RangeSet<TypedKey> visit(LogicalBinaryExpression node, Void context) {
RangeSet<TypedKey> leftRanges = node.getLeft().accept(this, context);
RangeSet<TypedKey> rightRanges = node.getRight().accept(this, context);
switch (node.getCompType()) {
case AND:
for (Range<TypedKey> range : leftRanges.asRanges()) {
rightRanges = rightRanges.subRangeSet(range);
}
break;
case OR:
rightRanges.addAll(leftRanges);
break;
case XOR:
// AND
RangeSet<TypedKey> intersection = rightRanges;
for (Range<TypedKey> range : leftRanges.asRanges()) {
intersection = intersection.subRangeSet(range);
}
// full set
rightRanges.addAll(leftRanges);
rightRanges.removeAll(intersection);
break;
default:
throwOnError(node);
}
return rightRanges;
}
@Override
protected RangeSet<TypedKey> visit(ComparisonBinaryExpression node, Void context) {
NormalizedPredicate predicate = node.normalize();
if (predicate == null) {
throwOnError(node);
}
// In order to match a prefix index, we have to cut the literal by prefix length.
// e.g., for table t:
// CREATE TABLE `t` {
// `b` VARCHAR(10) DEFAULT NULL,
// KEY `prefix_index` (`b`(2))
// }
//
// b(2) > "bbc" -> ["bb", +∞)
// b(2) >= "bbc" -> ["bb", +∞)
// b(2) < "bbc" -> (-∞, "bb"]
// b(2) <= "bbc" -> (-∞, "bb"]
// b(2) = "bbc" -> ["bb", "bb"]
// b(2) > "b" -> ["b", +∞)
// b(2) >= "b" -> ["b", +∞)
// b(2) < "b" -> (-∞, "b"]
// b(2) <= "b" -> (-∞, "b"]
//
// For varchar, `b`(2) will take first two characters(bytes) as prefix index.
// TODO: Note that TiDB only supports UTF-8, we need to check if prefix index behave differently
// under other encoding methods
int prefixLen = lengths.getOrDefault(predicate.getColumnRef(), DataType.UNSPECIFIED_LEN);
TypedKey literal = predicate.getTypedLiteral(prefixLen);
RangeSet<TypedKey> ranges = TreeRangeSet.create();
if (prefixLen != DataType.UNSPECIFIED_LEN) {
// With prefix length specified, the filter is loosen and so should the ranges
switch (predicate.getType()) {
case GREATER_THAN:
case GREATER_EQUAL:
ranges.add(Range.atLeast(literal));
break;
case LESS_THAN:
case LESS_EQUAL:
ranges.add(Range.atMost(literal));
break;
case EQUAL:
ranges.add(Range.singleton(literal));
break;
case NOT_EQUAL:
// Should return full range because prefix index predicate for NOT_EQUAL
// will be split into an NOT_EQUAL filter and a full range scan
ranges.add(Range.all());
break;
default:
throwOnError(node);
}
} else {
switch (predicate.getType()) {
case GREATER_THAN:
ranges.add(Range.greaterThan(literal));
break;
case GREATER_EQUAL:
ranges.add(Range.atLeast(literal));
break;
case LESS_THAN:
ranges.add(Range.lessThan(literal));
break;
case LESS_EQUAL:
ranges.add(Range.atMost(literal));
break;
case EQUAL:
ranges.add(Range.singleton(literal));
break;
case NOT_EQUAL:
ranges.add(Range.lessThan(literal));
ranges.add(Range.greaterThan(literal));
break;
default:
throwOnError(node);
}
}
return ranges;
}
@Override
protected RangeSet<TypedKey> visit(StringRegExpression node, Void context) {
ColumnRef columnRef = node.getColumnRef();
// In order to match a prefix index, we have to cut the literal by prefix length.
// e.g., for table t:
// CREATE TABLE `t` {
// `c1` VARCHAR(10) DEFAULT NULL,
// KEY `prefix_index` (`c`(2))
// }
// when the predicate is `c1` LIKE 'abc%', the index range should be ['ab', 'ab'].
// when the predicate is `c1` LIKE 'a%', the index range should be ['a', 'b').
// for varchar, `c1`(2) will take first two characters(bytes) as prefix index.
// TODO: Note that TiDB only supports UTF-8, we need to check if prefix index behave differently
// under other encoding methods
int prefixLen = lengths.getOrDefault(columnRef, DataType.UNSPECIFIED_LEN);
TypedKey literal = node.getTypedLiteral(prefixLen);
RangeSet<TypedKey> ranges = TreeRangeSet.create();
switch (node.getRegType()) {
case STARTS_WITH:
ranges.add(Range.atLeast(literal).intersection(Range.lessThan(literal.next(prefixLen))));
break;
default:
throwOnError(node);
}
return ranges;
}
}

55
src/main/java/com/pingcap/tikv/expression/visitor/MetaResolver.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression.visitor;
import com.pingcap.tikv.expression.ColumnRef;
import com.pingcap.tikv.expression.Expression;
import com.pingcap.tikv.meta.TiTableInfo;
import java.util.List;
import java.util.Objects;
public class MetaResolver extends DefaultVisitor<Void, Expression> {
public static void resolve(Expression expression, TiTableInfo table) {
MetaResolver resolver = new MetaResolver(table);
resolver.resolve(expression);
}
public static void resolve(List<? extends Expression> expressions, TiTableInfo table) {
MetaResolver resolver = new MetaResolver(table);
resolver.resolve(expressions);
}
private final TiTableInfo table;
public MetaResolver(TiTableInfo table) {
this.table = table;
}
public void resolve(List<? extends Expression> expressions) {
expressions.forEach(expression -> expression.accept(this, null));
}
public void resolve(Expression expression) {
Objects.requireNonNull(expression, "expression is null");
expression.accept(this, null);
}
@Override
protected Void visit(ColumnRef node, Expression parent) {
node.resolve(table);
return null;
}
}

315
src/main/java/com/pingcap/tikv/expression/visitor/ProtoConverter.java Normal file
View File

@ -0,0 +1,315 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression.visitor;
import static java.util.Objects.requireNonNull;
import com.google.common.collect.ImmutableMap;
import com.pingcap.tidb.tipb.Expr;
import com.pingcap.tidb.tipb.ExprType;
import com.pingcap.tidb.tipb.FieldType;
import com.pingcap.tidb.tipb.ScalarFuncSig;
import com.pingcap.tikv.codec.Codec.IntegerCodec;
import com.pingcap.tikv.codec.CodecDataOutput;
import com.pingcap.tikv.exception.TiExpressionException;
import com.pingcap.tikv.expression.*;
import com.pingcap.tikv.expression.AggregateFunction.FunctionType;
import com.pingcap.tikv.types.*;
import com.pingcap.tikv.types.DataType.EncodeType;
import java.util.IdentityHashMap;
import java.util.Map;
public class ProtoConverter extends Visitor<Expr, Object> {
// All concrete data type should be hooked to a type name
private static final Map<Class<? extends DataType>, String> SCALAR_SIG_MAP =
ImmutableMap.<Class<? extends DataType>, String>builder()
.put(IntegerType.class, "Int")
.put(BitType.class, "Int")
.put(DecimalType.class, "Decimal")
.put(RealType.class, "Real")
.put(DateTimeType.class, "Time")
.put(DateType.class, "Time")
.put(TimestampType.class, "Time")
.put(BytesType.class, "String")
.put(StringType.class, "String")
.build();
private final IdentityHashMap<Expression, DataType> typeMap;
private final boolean validateColPosition;
public ProtoConverter(IdentityHashMap<Expression, DataType> typeMap) {
this(typeMap, true);
}
/**
* Instantiate a {{@code ProtoConverter}} using a typeMap.
*
* @param typeMap the type map
* @param validateColPosition whether to consider column position in this converter. By default, a
* {{@code TiDAGRequest}} should check whether a {{@code ColumnRef}}'s position is correct in
* it's executors. Can ignore this validation if `validateColPosition` is set to false.
*/
public ProtoConverter(
IdentityHashMap<Expression, DataType> typeMap, boolean validateColPosition) {
this.typeMap = typeMap;
this.validateColPosition = validateColPosition;
}
private DataType getType(Expression expression) {
DataType type = typeMap.get(expression);
if (type == null) {
throw new TiExpressionException(String.format("Expression %s type unknown", expression));
}
return type;
}
private String getTypeSignature(Expression expression) {
DataType type = getType(expression);
String typeSignature = SCALAR_SIG_MAP.get(type.getClass());
if (typeSignature == null) {
throw new TiExpressionException(String.format("Type %s signature unknown", type));
}
return typeSignature;
}
public static Expr toProto(Expression expression) {
return toProto(expression, null);
}
public static Expr toProto(Expression expression, Object context) {
ExpressionTypeCoercer coercer = new ExpressionTypeCoercer();
coercer.infer(expression);
ProtoConverter converter = new ProtoConverter(coercer.getTypeMap());
return expression.accept(converter, context);
}
// Generate protobuf builder with partial data encoded.
// Scala Signature is left alone
private Expr.Builder scalaToPartialProto(Expression node, Object context) {
Expr.Builder builder = Expr.newBuilder();
// Scalar function type
builder.setTp(ExprType.ScalarFunc);
// Return type
builder.setFieldType(FieldType.newBuilder().setTp(getType(node).getTypeCode()).build());
for (Expression child : node.getChildren()) {
Expr exprProto = child.accept(this, context);
builder.addChildren(exprProto);
}
return builder;
}
@Override
protected Expr visit(LogicalBinaryExpression node, Object context) {
ScalarFuncSig protoSig;
switch (node.getCompType()) {
case AND:
protoSig = ScalarFuncSig.LogicalAnd;
break;
case OR:
protoSig = ScalarFuncSig.LogicalOr;
break;
case XOR:
protoSig = ScalarFuncSig.LogicalXor;
break;
default:
throw new TiExpressionException(
String.format("Unknown comparison type %s", node.getCompType()));
}
Expr.Builder builder = scalaToPartialProto(node, context);
builder.setSig(protoSig);
return builder.build();
}
@Override
protected Expr visit(ArithmeticBinaryExpression node, Object context) {
// assume after type coerce, children should be compatible
Expression child = node.getLeft();
String typeSignature = getTypeSignature(child);
ScalarFuncSig protoSig;
switch (node.getCompType()) {
// TODO: Add test for bitwise push down
case BIT_AND:
protoSig = ScalarFuncSig.BitAndSig;
break;
case BIT_OR:
protoSig = ScalarFuncSig.BitOrSig;
break;
case BIT_XOR:
protoSig = ScalarFuncSig.BitXorSig;
break;
case DIVIDE:
protoSig = ScalarFuncSig.valueOf("Divide" + typeSignature);
break;
case MINUS:
protoSig = ScalarFuncSig.valueOf("Minus" + typeSignature);
break;
case MULTIPLY:
protoSig = ScalarFuncSig.valueOf("Multiply" + typeSignature);
break;
case PLUS:
protoSig = ScalarFuncSig.valueOf("Plus" + typeSignature);
break;
default:
throw new TiExpressionException(
String.format("Unknown comparison type %s", node.getCompType()));
}
Expr.Builder builder = scalaToPartialProto(node, context);
builder.setSig(protoSig);
return builder.build();
}
@Override
protected Expr visit(ComparisonBinaryExpression node, Object context) {
// assume after type coerce, children should be compatible
Expression child = node.getLeft();
String typeSignature = getTypeSignature(child);
ScalarFuncSig protoSig;
switch (node.getComparisonType()) {
case EQUAL:
protoSig = ScalarFuncSig.valueOf("EQ" + typeSignature);
break;
case GREATER_EQUAL:
protoSig = ScalarFuncSig.valueOf("GE" + typeSignature);
break;
case GREATER_THAN:
protoSig = ScalarFuncSig.valueOf("GT" + typeSignature);
break;
case LESS_EQUAL:
protoSig = ScalarFuncSig.valueOf("LE" + typeSignature);
break;
case LESS_THAN:
protoSig = ScalarFuncSig.valueOf("LT" + typeSignature);
break;
case NOT_EQUAL:
protoSig = ScalarFuncSig.valueOf("NE" + typeSignature);
break;
default:
throw new TiExpressionException(
String.format("Unknown comparison type %s", node.getComparisonType()));
}
Expr.Builder builder = scalaToPartialProto(node, context);
builder.setSig(protoSig);
return builder.build();
}
@Override
protected Expr visit(StringRegExpression node, Object context) {
// assume after type coerce, children should be compatible
ScalarFuncSig protoSig;
switch (node.getRegType()) {
case STARTS_WITH:
case CONTAINS:
case ENDS_WITH:
case LIKE:
protoSig = ScalarFuncSig.LikeSig;
break;
default:
throw new TiExpressionException(String.format("Unknown reg type %s", node.getRegType()));
}
Expr.Builder builder = scalaToPartialProto(node, context);
builder.setSig(protoSig);
return builder.build();
}
@Override
@SuppressWarnings("unchecked")
protected Expr visit(ColumnRef node, Object context) {
long position = 0;
if (validateColPosition) {
requireNonNull(context, "Context of a ColumnRef should not be null");
Map<ColumnRef, Integer> colIdOffsetMap = (Map<ColumnRef, Integer>) context;
position =
requireNonNull(
colIdOffsetMap.get(node), "Required column position info is not in a valid context.");
}
Expr.Builder builder = Expr.newBuilder();
builder.setTp(ExprType.ColumnRef);
CodecDataOutput cdo = new CodecDataOutput();
// After switching to DAG request mode, expression value
// should be the index of table columns we provided in
// the first executor of a DAG request.
IntegerCodec.writeLong(cdo, position);
builder.setVal(cdo.toByteString());
return builder.build();
}
@Override
protected Expr visit(Constant node, Object context) {
Expr.Builder builder = Expr.newBuilder();
if (node.getValue() == null) {
builder.setTp(ExprType.Null);
return builder.build();
} else {
DataType type = node.getType();
builder.setTp(type.getProtoExprType());
CodecDataOutput cdo = new CodecDataOutput();
type.encode(cdo, EncodeType.PROTO, node.getValue());
builder.setVal(cdo.toByteString());
}
return builder.build();
}
@Override
protected Expr visit(AggregateFunction node, Object context) {
Expr.Builder builder = Expr.newBuilder();
FunctionType type = node.getType();
switch (type) {
case Max:
builder.setTp(ExprType.Max);
break;
case Sum:
builder.setTp(ExprType.Sum);
break;
case Min:
builder.setTp(ExprType.Min);
break;
case First:
builder.setTp(ExprType.First);
break;
case Count:
builder.setTp(ExprType.Count);
break;
}
for (Expression arg : node.getChildren()) {
Expr exprProto = arg.accept(this, context);
builder.addChildren(exprProto);
}
return builder.build();
}
@Override
protected Expr visit(IsNull node, Object context) {
String typeSignature = getTypeSignature(node.getExpression());
ScalarFuncSig protoSig = ScalarFuncSig.valueOf(typeSignature + "IsNull");
Expr.Builder builder = scalaToPartialProto(node, context);
builder.setSig(protoSig);
return builder.build();
}
@Override
protected Expr visit(Not node, Object context) {
ScalarFuncSig protoSig = ScalarFuncSig.UnaryNot;
Expr.Builder builder = scalaToPartialProto(node, context);
builder.setSig(protoSig);
return builder.build();
}
}

65
src/main/java/com/pingcap/tikv/expression/visitor/PseudoCostCalculator.java Normal file
View File

@ -0,0 +1,65 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression.visitor;
import com.pingcap.tikv.expression.ComparisonBinaryExpression;
import com.pingcap.tikv.expression.Expression;
import com.pingcap.tikv.expression.LogicalBinaryExpression;
public class PseudoCostCalculator extends DefaultVisitor<Double, Void> {
public static double calculateCost(Expression expr) {
PseudoCostCalculator calc = new PseudoCostCalculator();
return expr.accept(calc, null);
}
@Override
protected Double process(Expression node, Void context) {
return 1.0;
}
@Override
protected Double visit(LogicalBinaryExpression node, Void context) {
double leftCost = node.getLeft().accept(this, context);
double rightCost = node.getLeft().accept(this, context);
switch (node.getCompType()) {
case AND:
return leftCost * rightCost;
case OR:
case XOR:
return leftCost + rightCost;
default:
return 1.0;
}
}
@Override
protected Double visit(ComparisonBinaryExpression node, Void context) {
switch (node.getComparisonType()) {
case EQUAL:
return 0.01;
case GREATER_EQUAL:
case GREATER_THAN:
case LESS_EQUAL:
case LESS_THAN:
// magic number for testing
return 0.3;
case NOT_EQUAL:
return 0.99;
default:
return 1.0;
}
}
}

53
src/main/java/com/pingcap/tikv/expression/visitor/SupportedExpressionValidator.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.expression.visitor;
import com.pingcap.tikv.expression.Expression;
import com.pingcap.tikv.expression.ExpressionBlacklist;
public class SupportedExpressionValidator extends DefaultVisitor<Boolean, ExpressionBlacklist> {
private static final SupportedExpressionValidator validator = new SupportedExpressionValidator();
public static boolean isSupportedExpression(Expression node, ExpressionBlacklist blacklist) {
if (!node.accept(validator, blacklist)) {
return false;
}
try {
ExpressionTypeCoercer coercer = new ExpressionTypeCoercer();
coercer.infer(node);
ProtoConverter protoConverter = new ProtoConverter(coercer.getTypeMap(), false);
if (node.accept(protoConverter, null) == null) {
return false;
}
} catch (Exception e) {
return false;
}
return true;
}
@Override
protected Boolean process(Expression node, ExpressionBlacklist blacklist) {
if (blacklist != null && blacklist.isUnsupportedPushdownExpr(getClass())) {
return false;
}
for (Expression expr : node.getChildren()) {
if (!expr.accept(this, blacklist)) {
return false;
}
}
return true;
}
}

78
src/main/java/com/pingcap/tikv/key/CompoundKey.java Normal file
View File

@ -0,0 +1,78 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.key;
import com.google.common.base.Joiner;
import com.pingcap.tikv.codec.CodecDataOutput;
import java.util.ArrayList;
import java.util.List;
public class CompoundKey extends Key {
private final List<Key> keys;
protected CompoundKey(List<Key> keys, byte[] value) {
super(value);
this.keys = keys;
}
public static CompoundKey concat(Key lKey, Key rKey) {
Builder builder = newBuilder();
builder.append(lKey).append(rKey);
return builder.build();
}
public List<Key> getKeys() {
return keys;
}
public static Builder newBuilder() {
return new Builder();
}
public static class Builder {
private final List<Key> keys = new ArrayList<>();
public Builder append(Key key) {
if (key instanceof CompoundKey) {
CompoundKey compKey = (CompoundKey) key;
for (Key child : compKey.getKeys()) {
append(child);
}
} else {
keys.add(key);
}
return this;
}
public CompoundKey build() {
int totalLen = 0;
for (Key key : keys) {
totalLen += key.getBytes().length;
}
CodecDataOutput cdo = new CodecDataOutput(totalLen);
for (Key key : keys) {
cdo.write(key.getBytes());
}
return new CompoundKey(keys, cdo.toBytes());
}
}
@Override
public String toString() {
return String.format("[%s]", Joiner.on(",").useForNull("Null").join(keys));
}
}

72
src/main/java/com/pingcap/tikv/key/IndexKey.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.key;
import com.google.common.base.Joiner;
import com.pingcap.tikv.codec.Codec.IntegerCodec;
import com.pingcap.tikv.codec.CodecDataOutput;
import com.pingcap.tikv.exception.TypeException;
public class IndexKey extends Key {
private static final byte[] IDX_PREFIX_SEP = new byte[] {'_', 'i'};
private final long tableId;
private final long indexId;
private final Key[] dataKeys;
private IndexKey(long tableId, long indexId, Key[] dataKeys) {
super(encode(tableId, indexId, dataKeys));
this.tableId = tableId;
this.indexId = indexId;
this.dataKeys = dataKeys;
}
public static IndexKey toIndexKey(long tableId, long indexId, Key... dataKeys) {
return new IndexKey(tableId, indexId, dataKeys);
}
private static byte[] encode(long tableId, long indexId, Key[] dataKeys) {
CodecDataOutput cdo = new CodecDataOutput();
cdo.write(TBL_PREFIX);
IntegerCodec.writeLong(cdo, tableId);
cdo.write(IDX_PREFIX_SEP);
IntegerCodec.writeLong(cdo, indexId);
for (Key key : dataKeys) {
if (key == null) {
throw new TypeException("key cannot be null");
}
cdo.write(key.getBytes());
}
return cdo.toBytes();
}
public long getTableId() {
return tableId;
}
public long getIndexId() {
return indexId;
}
public Key[] getDataKeys() {
return dataKeys;
}
@Override
public String toString() {
return String.format("[%s]", Joiner.on(",").useForNull("null").join(dataKeys));
}
}

184
src/main/java/com/pingcap/tikv/key/Key.java Normal file
View File

@ -0,0 +1,184 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.key;
import static com.pingcap.tikv.codec.KeyUtils.formatBytes;
import static java.util.Objects.requireNonNull;
import com.google.protobuf.ByteString;
import com.pingcap.tikv.codec.CodecDataOutput;
import com.pingcap.tikv.types.DataType;
import com.pingcap.tikv.util.FastByteComparisons;
import java.util.Arrays;
public class Key implements Comparable<Key> {
protected static final byte[] TBL_PREFIX = new byte[] {'t'};
protected final byte[] value;
protected final int infFlag;
public static final Key EMPTY = createEmpty();
public static final Key NULL = createNull();
public static final Key MIN = createTypelessMin();
public static final Key MAX = createTypelessMax();
private Key(byte[] value, boolean negative) {
this.value = requireNonNull(value, "value is null");
this.infFlag = (value.length == 0 ? 1 : 0) * (negative ? -1 : 1);
}
protected Key(byte[] value) {
this(value, false);
}
public static Key toRawKey(ByteString bytes, boolean negative) {
return new Key(bytes.toByteArray(), negative);
}
public static Key toRawKey(ByteString bytes) {
return new Key(bytes.toByteArray());
}
public static Key toRawKey(byte[] bytes, boolean negative) {
return new Key(bytes, negative);
}
public static Key toRawKey(byte[] bytes) {
return new Key(bytes);
}
private static Key createNull() {
CodecDataOutput cdo = new CodecDataOutput();
DataType.encodeNull(cdo);
return new Key(cdo.toBytes()) {
@Override
public String toString() {
return "null";
}
};
}
private static Key createEmpty() {
return new Key(new byte[0]) {
@Override
public Key next() {
return this;
}
@Override
public String toString() {
return "EMPTY";
}
};
}
private static Key createTypelessMin() {
CodecDataOutput cdo = new CodecDataOutput();
DataType.encodeIndex(cdo);
return new Key(cdo.toBytes()) {
@Override
public String toString() {
return "MIN";
}
};
}
private static Key createTypelessMax() {
CodecDataOutput cdo = new CodecDataOutput();
DataType.encodeMaxValue(cdo);
return new Key(cdo.toBytes()) {
@Override
public String toString() {
return "MAX";
}
};
}
/**
* The next key for bytes domain It first plus one at LSB and if LSB overflows, a zero byte is
* appended at the end Original bytes will be reused if possible
*
* @return encoded results
*/
public Key next() {
return toRawKey(nextValue(value));
}
static byte[] nextValue(byte[] value) {
int i;
byte[] newVal = Arrays.copyOf(value, value.length);
for (i = newVal.length - 1; i >= 0; i--) {
newVal[i]++;
if (newVal[i] != 0) {
break;
}
}
if (i == -1) {
return Arrays.copyOf(value, value.length + 1);
} else {
return newVal;
}
}
@Override
public int compareTo(Key other) {
requireNonNull(other, "other is null");
if ((this.infFlag | other.infFlag) != 0) {
return this.infFlag - other.infFlag;
}
return FastByteComparisons.compareTo(value, other.value);
}
@Override
public boolean equals(Object other) {
if (other == this) {
return true;
}
if (other instanceof Key) {
return compareTo((Key) other) == 0;
} else {
return false;
}
}
@Override
public int hashCode() {
return Arrays.hashCode(value) * infFlag;
}
public byte[] getBytes() {
return value;
}
public ByteString toByteString() {
return ByteString.copyFrom(value);
}
public int getInfFlag() {
return infFlag;
}
@Override
public String toString() {
if (infFlag < 0) {
return "-INF";
} else if (infFlag > 0) {
return "+INF";
} else {
return String.format("{%s}", formatBytes(value));
}
}
}

176
src/main/java/com/pingcap/tikv/key/RowKey.java Normal file
View File

@ -0,0 +1,176 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.key;
import static com.pingcap.tikv.codec.Codec.IntegerCodec.writeLong;
import com.pingcap.tikv.codec.Codec.IntegerCodec;
import com.pingcap.tikv.codec.CodecDataInput;
import com.pingcap.tikv.codec.CodecDataOutput;
import com.pingcap.tikv.exception.TiClientInternalException;
import com.pingcap.tikv.exception.TiExpressionException;
import com.pingcap.tikv.key.RowKey.DecodeResult.Status;
import com.pingcap.tikv.util.FastByteComparisons;
import java.util.Objects;
public class RowKey extends Key {
private static final byte[] REC_PREFIX_SEP = new byte[] {'_', 'r'};
private final long tableId;
private final long handle;
private final boolean maxHandleFlag;
private RowKey(long tableId, long handle) {
super(encode(tableId, handle));
this.tableId = tableId;
this.handle = handle;
this.maxHandleFlag = false;
}
/**
* The RowKey indicating maximum handle (its value exceeds Long.Max_Value)
*
* <p>Initializes an imaginary globally MAXIMUM rowKey with tableId.
*/
private RowKey(long tableId) {
super(encodeBeyondMaxHandle(tableId));
this.tableId = tableId;
this.handle = Long.MAX_VALUE;
this.maxHandleFlag = true;
}
public static RowKey toRowKey(long tableId, long handle) {
return new RowKey(tableId, handle);
}
public static RowKey toRowKey(long tableId, TypedKey handle) {
Object obj = handle.getValue();
if (obj instanceof Long) {
return new RowKey(tableId, (long) obj);
}
throw new TiExpressionException("Cannot encode row key with non-long type");
}
public static RowKey createMin(long tableId) {
return toRowKey(tableId, Long.MIN_VALUE);
}
public static RowKey createBeyondMax(long tableId) {
return new RowKey(tableId);
}
private static byte[] encode(long tableId, long handle) {
CodecDataOutput cdo = new CodecDataOutput();
encodePrefix(cdo, tableId);
writeLong(cdo, handle);
return cdo.toBytes();
}
private static byte[] encodeBeyondMaxHandle(long tableId) {
return nextValue(encode(tableId, Long.MAX_VALUE));
}
@Override
public RowKey next() {
long handle = getHandle();
boolean maxHandleFlag = getMaxHandleFlag();
if (maxHandleFlag) {
throw new TiClientInternalException("Handle overflow for Long MAX");
}
if (handle == Long.MAX_VALUE) {
return createBeyondMax(tableId);
}
return new RowKey(tableId, handle + 1);
}
public long getTableId() {
return tableId;
}
public long getHandle() {
return handle;
}
private boolean getMaxHandleFlag() {
return maxHandleFlag;
}
@Override
public String toString() {
return Long.toString(handle);
}
private static void encodePrefix(CodecDataOutput cdo, long tableId) {
cdo.write(TBL_PREFIX);
writeLong(cdo, tableId);
cdo.write(REC_PREFIX_SEP);
}
public static class DecodeResult {
public long handle;
public enum Status {
MIN,
MAX,
EQUAL,
LESS,
GREATER,
UNKNOWN_INF
}
public Status status;
}
public static void tryDecodeRowKey(long tableId, byte[] rowKey, DecodeResult outResult) {
Objects.requireNonNull(rowKey, "rowKey cannot be null");
if (rowKey.length == 0) {
outResult.status = Status.UNKNOWN_INF;
return;
}
CodecDataOutput cdo = new CodecDataOutput();
encodePrefix(cdo, tableId);
byte[] tablePrefix = cdo.toBytes();
int res =
FastByteComparisons.compareTo(
tablePrefix,
0,
tablePrefix.length,
rowKey,
0,
Math.min(rowKey.length, tablePrefix.length));
if (res > 0) {
outResult.status = Status.MIN;
return;
}
if (res < 0) {
outResult.status = Status.MAX;
return;
}
CodecDataInput cdi = new CodecDataInput(rowKey);
cdi.skipBytes(tablePrefix.length);
if (cdi.available() == 8) {
outResult.status = Status.EQUAL;
} else if (cdi.available() < 8) {
outResult.status = Status.LESS;
} else if (cdi.available() > 8) {
outResult.status = Status.GREATER;
}
outResult.handle = IntegerCodec.readPartialLong(cdi);
}
}

84
src/main/java/com/pingcap/tikv/key/TypedKey.java Normal file
View File

@ -0,0 +1,84 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.key;
import static java.util.Objects.requireNonNull;
import com.pingcap.tikv.codec.CodecDataInput;
import com.pingcap.tikv.codec.CodecDataOutput;
import com.pingcap.tikv.exception.TypeException;
import com.pingcap.tikv.types.DataType;
public class TypedKey extends Key {
private final DataType type;
public TypedKey(Object val, DataType type, int prefixLength) {
super(encodeKey(val, type, prefixLength));
this.type = type;
}
public DataType getType() {
return type;
}
public Object getValue() {
CodecDataInput cdi = new CodecDataInput(value);
return type.decode(cdi);
}
public static TypedKey toTypedKey(Object val, DataType type) {
return toTypedKey(val, type, DataType.UNSPECIFIED_LEN);
}
/**
* Map a typed value into TypedKey, only encoding first prefixLength bytes When prefixLength is
* DataType.UNSPECIFIED_LEN, encode full length of value
*
* @param val value
* @param type type of value
* @param prefixLength described above
* @return an encoded TypedKey
*/
public static TypedKey toTypedKey(Object val, DataType type, int prefixLength) {
requireNonNull(type, "type is null");
return new TypedKey(val, type, prefixLength);
}
private static byte[] encodeKey(Object val, DataType type, int prefixLength) {
CodecDataOutput cdo = new CodecDataOutput();
type.encodeKey(cdo, val, type, prefixLength);
return cdo.toBytes();
}
public TypedKey next(int prefixLength) {
Object val = getValue();
if (val instanceof String) {
return toTypedKey(nextValue(((String) val).getBytes()), type, prefixLength);
} else if (val instanceof byte[]) {
return toTypedKey(nextValue(((byte[]) val)), type, prefixLength);
} else {
throw new TypeException(
"Type for TypedKey in next() function must be either String or Byte array");
}
}
@Override
public String toString() {
CodecDataInput cdi = new CodecDataInput(value);
Object val = type.decode(cdi);
return String.format("%s", val);
}
}

45
src/main/java/com/pingcap/tikv/meta/CIStr.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.meta;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
/** This class is mapping TiDB's CIStr/ For internal use only. */
@JsonIgnoreProperties(ignoreUnknown = true)
public class CIStr {
private final String o; // original
private final String l;
@JsonCreator
private CIStr(@JsonProperty("O") String o, @JsonProperty("L") String l) {
this.o = o;
this.l = l;
}
public static CIStr newCIStr(String str) {
return new CIStr(str, str.toLowerCase());
}
public String getO() {
return o;
}
public String getL() {
return l;
}
}

273
src/main/java/com/pingcap/tikv/meta/Collation.java Normal file
View File

@ -0,0 +1,273 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.meta;
import com.google.common.collect.ImmutableMap;
import java.util.Map;
public class Collation {
public static int translate(String collation) {
Integer code = collationMap.get(collation);
if (code == null) {
return DEF_COLLATION_CODE;
}
return code;
}
public static String translate(int code) {
String collation = collationCodeMap.get(code);
if (collation == null) {
return "";
}
return collation;
}
public static final int DEF_COLLATION_CODE = 83;
private static final Map<String, Integer> collationMap;
private static final Map<Integer, String> collationCodeMap;
static {
collationMap =
ImmutableMap.<String, Integer>builder()
.put("big5_chinese_ci", 1)
.put("latin2_czech_cs", 2)
.put("dec8_swedish_ci", 3)
.put("cp850_general_ci", 4)
.put("latin1_german1_ci", 5)
.put("hp8_english_ci", 6)
.put("koi8r_general_ci", 7)
.put("latin1_swedish_ci", 8)
.put("latin2_general_ci", 9)
.put("swe7_swedish_ci", 10)
.put("ascii_general_ci", 11)
.put("ujis_japanese_ci", 12)
.put("sjis_japanese_ci", 13)
.put("cp1251_bulgarian_ci", 14)
.put("latin1_danish_ci", 15)
.put("hebrew_general_ci", 16)
.put("tis620_thai_ci", 18)
.put("euckr_korean_ci", 19)
.put("latin7_estonian_cs", 20)
.put("latin2_hungarian_ci", 21)
.put("koi8u_general_ci", 22)
.put("cp1251_ukrainian_ci", 23)
.put("gb2312_chinese_ci", 24)
.put("greek_general_ci", 25)
.put("cp1250_general_ci", 26)
.put("latin2_croatian_ci", 27)
.put("gbk_chinese_ci", 28)
.put("cp1257_lithuanian_ci", 29)
.put("latin5_turkish_ci", 30)
.put("latin1_german2_ci", 31)
.put("armscii8_general_ci", 32)
.put("utf8_general_ci", 33)
.put("cp1250_czech_cs", 34)
.put("ucs2_general_ci", 35)
.put("cp866_general_ci", 36)
.put("keybcs2_general_ci", 37)
.put("macce_general_ci", 38)
.put("macroman_general_ci", 39)
.put("cp852_general_ci", 40)
.put("latin7_general_ci", 41)
.put("latin7_general_cs", 42)
.put("macce_bin", 43)
.put("cp1250_croatian_ci", 44)
.put("utf8mb4_general_ci", 45)
.put("utf8mb4_bin", 46)
.put("latin1_bin", 47)
.put("latin1_general_ci", 48)
.put("latin1_general_cs", 49)
.put("cp1251_bin", 50)
.put("cp1251_general_ci", 51)
.put("cp1251_general_cs", 52)
.put("macroman_bin", 53)
.put("utf16_general_ci", 54)
.put("utf16_bin", 55)
.put("utf16le_general_ci", 56)
.put("cp1256_general_ci", 57)
.put("cp1257_bin", 58)
.put("cp1257_general_ci", 59)
.put("utf32_general_ci", 60)
.put("utf32_bin", 61)
.put("utf16le_bin", 62)
.put("binary", 63)
.put("armscii8_bin", 64)
.put("ascii_bin", 65)
.put("cp1250_bin", 66)
.put("cp1256_bin", 67)
.put("cp866_bin", 68)
.put("dec8_bin", 69)
.put("greek_bin", 70)
.put("hebrew_bin", 71)
.put("hp8_bin", 72)
.put("keybcs2_bin", 73)
.put("koi8r_bin", 74)
.put("koi8u_bin", 75)
.put("latin2_bin", 77)
.put("latin5_bin", 78)
.put("latin7_bin", 79)
.put("cp850_bin", 80)
.put("cp852_bin", 81)
.put("swe7_bin", 82)
.put("utf8_bin", 83)
.put("big5_bin", 84)
.put("euckr_bin", 85)
.put("gb2312_bin", 86)
.put("gbk_bin", 87)
.put("sjis_bin", 88)
.put("tis620_bin", 89)
.put("ucs2_bin", 90)
.put("ujis_bin", 91)
.put("geostd8_general_ci", 92)
.put("geostd8_bin", 93)
.put("latin1_spanish_ci", 94)
.put("cp932_japanese_ci", 95)
.put("cp932_bin", 96)
.put("eucjpms_japanese_ci", 97)
.put("eucjpms_bin", 98)
.put("cp1250_polish_ci", 99)
.put("utf16_unicode_ci", 101)
.put("utf16_icelandic_ci", 102)
.put("utf16_latvian_ci", 103)
.put("utf16_romanian_ci", 104)
.put("utf16_slovenian_ci", 105)
.put("utf16_polish_ci", 106)
.put("utf16_estonian_ci", 107)
.put("utf16_spanish_ci", 108)
.put("utf16_swedish_ci", 109)
.put("utf16_turkish_ci", 110)
.put("utf16_czech_ci", 111)
.put("utf16_danish_ci", 112)
.put("utf16_lithuanian_ci", 113)
.put("utf16_slovak_ci", 114)
.put("utf16_spanish2_ci", 115)
.put("utf16_roman_ci", 116)
.put("utf16_persian_ci", 117)
.put("utf16_esperanto_ci", 118)
.put("utf16_hungarian_ci", 119)
.put("utf16_sinhala_ci", 120)
.put("utf16_german2_ci", 121)
.put("utf16_croatian_ci", 122)
.put("utf16_unicode_520_ci", 123)
.put("utf16_vietnamese_ci", 124)
.put("ucs2_unicode_ci", 128)
.put("ucs2_icelandic_ci", 129)
.put("ucs2_latvian_ci", 130)
.put("ucs2_romanian_ci", 131)
.put("ucs2_slovenian_ci", 132)
.put("ucs2_polish_ci", 133)
.put("ucs2_estonian_ci", 134)
.put("ucs2_spanish_ci", 135)
.put("ucs2_swedish_ci", 136)
.put("ucs2_turkish_ci", 137)
.put("ucs2_czech_ci", 138)
.put("ucs2_danish_ci", 139)
.put("ucs2_lithuanian_ci", 140)
.put("ucs2_slovak_ci", 141)
.put("ucs2_spanish2_ci", 142)
.put("ucs2_roman_ci", 143)
.put("ucs2_persian_ci", 144)
.put("ucs2_esperanto_ci", 145)
.put("ucs2_hungarian_ci", 146)
.put("ucs2_sinhala_ci", 147)
.put("ucs2_german2_ci", 148)
.put("ucs2_croatian_ci", 149)
.put("ucs2_unicode_520_ci", 150)
.put("ucs2_vietnamese_ci", 151)
.put("ucs2_general_mysql500_ci", 159)
.put("utf32_unicode_ci", 160)
.put("utf32_icelandic_ci", 161)
.put("utf32_latvian_ci", 162)
.put("utf32_romanian_ci", 163)
.put("utf32_slovenian_ci", 164)
.put("utf32_polish_ci", 165)
.put("utf32_estonian_ci", 166)
.put("utf32_spanish_ci", 167)
.put("utf32_swedish_ci", 168)
.put("utf32_turkish_ci", 169)
.put("utf32_czech_ci", 170)
.put("utf32_danish_ci", 171)
.put("utf32_lithuanian_ci", 172)
.put("utf32_slovak_ci", 173)
.put("utf32_spanish2_ci", 174)
.put("utf32_roman_ci", 175)
.put("utf32_persian_ci", 176)
.put("utf32_esperanto_ci", 177)
.put("utf32_hungarian_ci", 178)
.put("utf32_sinhala_ci", 179)
.put("utf32_german2_ci", 180)
.put("utf32_croatian_ci", 181)
.put("utf32_unicode_520_ci", 182)
.put("utf32_vietnamese_ci", 183)
.put("utf8_unicode_ci", 192)
.put("utf8_icelandic_ci", 193)
.put("utf8_latvian_ci", 194)
.put("utf8_romanian_ci", 195)
.put("utf8_slovenian_ci", 196)
.put("utf8_polish_ci", 197)
.put("utf8_estonian_ci", 198)
.put("utf8_spanish_ci", 199)
.put("utf8_swedish_ci", 200)
.put("utf8_turkish_ci", 201)
.put("utf8_czech_ci", 202)
.put("utf8_danish_ci", 203)
.put("utf8_lithuanian_ci", 204)
.put("utf8_slovak_ci", 205)
.put("utf8_spanish2_ci", 206)
.put("utf8_roman_ci", 207)
.put("utf8_persian_ci", 208)
.put("utf8_esperanto_ci", 209)
.put("utf8_hungarian_ci", 210)
.put("utf8_sinhala_ci", 211)
.put("utf8_german2_ci", 212)
.put("utf8_croatian_ci", 213)
.put("utf8_unicode_520_ci", 214)
.put("utf8_vietnamese_ci", 215)
.put("utf8_general_mysql500_ci", 223)
.put("utf8mb4_unicode_ci", 224)
.put("utf8mb4_icelandic_ci", 225)
.put("utf8mb4_latvian_ci", 226)
.put("utf8mb4_romanian_ci", 227)
.put("utf8mb4_slovenian_ci", 228)
.put("utf8mb4_polish_ci", 229)
.put("utf8mb4_estonian_ci", 230)
.put("utf8mb4_spanish_ci", 231)
.put("utf8mb4_swedish_ci", 232)
.put("utf8mb4_turkish_ci", 233)
.put("utf8mb4_czech_ci", 234)
.put("utf8mb4_danish_ci", 235)
.put("utf8mb4_lithuanian_ci", 236)
.put("utf8mb4_slovak_ci", 237)
.put("utf8mb4_spanish2_ci", 238)
.put("utf8mb4_roman_ci", 239)
.put("utf8mb4_persian_ci", 240)
.put("utf8mb4_esperanto_ci", 241)
.put("utf8mb4_hungarian_ci", 242)
.put("utf8mb4_sinhala_ci", 243)
.put("utf8mb4_german2_ci", 244)
.put("utf8mb4_croatian_ci", 245)
.put("utf8mb4_unicode_520_ci", 246)
.put("utf8mb4_vietnamese_ci", 247)
.build();
ImmutableMap.Builder<Integer, String> builder = ImmutableMap.builder();
for (String collation : collationMap.keySet()) {
builder.put(collationMap.get(collation), collation);
}
collationCodeMap = builder.build();
}
}

55
src/main/java/com/pingcap/tikv/meta/IndexType.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.meta;
import com.pingcap.tikv.exception.TiClientInternalException;
// Actually we are not using either real btree or hash index
// TiDB has its own way for indexing as key value pair.
public enum IndexType {
IndexTypeInvalid(0),
IndexTypeBtree(1),
IndexTypeHash(2);
private final int type;
IndexType(int type) {
this.type = type;
}
public static IndexType fromValue(int type) {
for (IndexType e : IndexType.values()) {
if (e.type == type) {
return e;
}
}
throw new TiClientInternalException("Invalid index type code: " + type);
}
public int getTypeCode() {
return type;
}
public String toString() {
switch (this.type) {
case 1:
return "BTREE";
case 2:
return "HASH";
}
return "Invalid";
}
}

46
src/main/java/com/pingcap/tikv/meta/SchemaState.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.meta;
import com.pingcap.tikv.exception.TiClientInternalException;
public enum SchemaState {
StateNone(0),
StateDeleteOnly(1),
StateWriteOnly(2),
StateWriteReorganization(3),
StateDeleteReorganization(4),
StatePublic(5);
private final int state;
SchemaState(int state) {
this.state = state;
}
public static SchemaState fromValue(int b) {
for (SchemaState e : SchemaState.values()) {
if (e.state == b) {
return e;
}
}
throw new TiClientInternalException("Invalid SchemaState code: " + b);
}
public int getStateCode() {
return state;
}
}

346
src/main/java/com/pingcap/tikv/meta/TiColumnInfo.java Normal file
View File

@ -0,0 +1,346 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.meta;
import static java.util.Objects.requireNonNull;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
import com.pingcap.tidb.tipb.ColumnInfo;
import com.pingcap.tikv.codec.CodecDataOutput;
import com.pingcap.tikv.types.DataType;
import com.pingcap.tikv.types.DataType.EncodeType;
import com.pingcap.tikv.types.DataTypeFactory;
import com.pingcap.tikv.types.IntegerType;
import java.io.Serializable;
import java.util.List;
import java.util.Objects;
@JsonIgnoreProperties(ignoreUnknown = true)
public class TiColumnInfo implements Serializable {
private final long id;
private final String name;
private final int offset;
private final DataType type;
private final SchemaState schemaState;
private final String comment;
private final boolean isPrimaryKey;
private final String defaultValue;
private final String originDefaultValue;
public static TiColumnInfo getRowIdColumn(int offset) {
return new TiColumnInfo(-1, "_tidb_rowid", offset, IntegerType.ROW_ID_TYPE, true);
}
@VisibleForTesting private static final int PK_MASK = 0x2;
@JsonCreator
public TiColumnInfo(
@JsonProperty("id") long id,
@JsonProperty("name") CIStr name,
@JsonProperty("offset") int offset,
@JsonProperty("type") InternalTypeHolder type,
@JsonProperty("state") int schemaState,
@JsonProperty("origin_default") String originalDefaultValue,
@JsonProperty("default") String defaultValue,
@JsonProperty("comment") String comment) {
this.id = id;
this.name = requireNonNull(name, "column name is null").getL();
this.offset = offset;
this.type = DataTypeFactory.of(requireNonNull(type, "type is null"));
this.schemaState = SchemaState.fromValue(schemaState);
this.comment = comment;
this.defaultValue = defaultValue;
this.originDefaultValue = originalDefaultValue;
// I don't think pk flag should be set on type
// Refactor against original tidb code
this.isPrimaryKey = (type.getFlag() & PK_MASK) > 0;
}
public TiColumnInfo(
long id,
String name,
int offset,
DataType type,
SchemaState schemaState,
String originalDefaultValue,
String defaultValue,
String comment) {
this.id = id;
this.name = requireNonNull(name, "column name is null").toLowerCase();
this.offset = offset;
this.type = requireNonNull(type, "data type is null");
this.schemaState = schemaState;
this.comment = comment;
this.defaultValue = defaultValue;
this.originDefaultValue = originalDefaultValue;
this.isPrimaryKey = (type.getFlag() & PK_MASK) > 0;
}
public TiColumnInfo copyWithoutPrimaryKey() {
InternalTypeHolder typeHolder = type.toTypeHolder();
typeHolder.setFlag(type.getFlag() & (~TiColumnInfo.PK_MASK));
DataType newType = DataTypeFactory.of(typeHolder);
return new TiColumnInfo(
this.id,
this.name,
this.offset,
newType,
this.schemaState,
this.originDefaultValue,
this.defaultValue,
this.comment);
}
@VisibleForTesting
public TiColumnInfo(long id, String name, int offset, DataType type, boolean isPrimaryKey) {
this.id = id;
this.name = requireNonNull(name, "column name is null").toLowerCase();
this.offset = offset;
this.type = requireNonNull(type, "data type is null");
this.schemaState = SchemaState.StatePublic;
this.comment = "";
this.isPrimaryKey = isPrimaryKey;
this.originDefaultValue = "1";
this.defaultValue = "";
}
public long getId() {
return this.id;
}
public String getName() {
return this.name;
}
public boolean matchName(String name) {
return this.name.equalsIgnoreCase(name);
}
public int getOffset() {
return this.offset;
}
public DataType getType() {
return type;
}
public SchemaState getSchemaState() {
return schemaState;
}
public String getComment() {
return comment;
}
public boolean isPrimaryKey() {
return isPrimaryKey;
}
public String getDefaultValue() {
return defaultValue;
}
public String getOriginDefaultValue() {
return originDefaultValue;
}
public ByteString getOriginDefaultValueAsByteString() {
CodecDataOutput cdo = new CodecDataOutput();
type.encode(cdo, EncodeType.VALUE, type.getOriginDefaultValue(originDefaultValue));
return cdo.toByteString();
}
@JsonIgnoreProperties(ignoreUnknown = true)
public static class InternalTypeHolder {
private int tp;
private int flag;
private long flen;
private int decimal;
private String charset;
private String collate;
private String defaultValue;
private String originDefaultValue;
private List<String> elems;
public void setTp(int tp) {
this.tp = tp;
}
public void setFlag(int flag) {
this.flag = flag;
}
public void setFlen(long flen) {
this.flen = flen;
}
public void setDecimal(int decimal) {
this.decimal = decimal;
}
public void setCharset(String charset) {
this.charset = charset;
}
public void setCollate(String collate) {
this.collate = collate;
}
public void setDefaultValue(String defaultValue) {
this.defaultValue = defaultValue;
}
public void setOriginDefaultValue(String originDefaultValue) {
this.originDefaultValue = originDefaultValue;
}
public void setElems(List<String> elems) {
this.elems = elems;
}
interface Builder<E extends DataType> {
E build(InternalTypeHolder holder);
}
@JsonCreator
public InternalTypeHolder(
@JsonProperty("Tp") int tp,
@JsonProperty("Flag") int flag,
@JsonProperty("Flen") long flen,
@JsonProperty("Decimal") int decimal,
@JsonProperty("Charset") String charset,
@JsonProperty("origin_default") String originalDefaultValue,
@JsonProperty("default") String defaultValue,
@JsonProperty("Collate") String collate,
@JsonProperty("Elems") List<String> elems) {
this.tp = tp;
this.flag = flag;
this.flen = flen;
this.decimal = decimal;
this.charset = charset;
this.collate = collate;
this.defaultValue = defaultValue;
this.originDefaultValue = originalDefaultValue;
this.elems = elems;
}
public InternalTypeHolder(ColumnInfo c) {
this.tp = c.getTp();
this.flag = c.getFlag();
this.flen = c.getColumnLen();
this.decimal = c.getDecimal();
this.charset = "";
this.collate = Collation.translate(c.getCollation());
this.elems = c.getElemsList();
this.defaultValue = c.getDefaultVal().toStringUtf8();
// TODO: we may need write a functon about get origin default value according to the string.
this.originDefaultValue = "";
}
public int getTp() {
return tp;
}
public int getFlag() {
return flag;
}
public long getFlen() {
return flen;
}
public int getDecimal() {
return decimal;
}
public String getCharset() {
return charset;
}
public String getCollate() {
return collate;
}
public List<String> getElems() {
return elems;
}
public String getDefaultValue() {
return defaultValue;
}
public String getOriginDefaultValue() {
return originDefaultValue;
}
}
TiIndexColumn toFakeIndexColumn() {
// we don't use original length of column since for a clustered index column
// it always full index instead of prefix index
return new TiIndexColumn(CIStr.newCIStr(getName()), getOffset(), DataType.UNSPECIFIED_LEN);
}
TiIndexColumn toIndexColumn() {
return new TiIndexColumn(CIStr.newCIStr(getName()), getOffset(), getType().getLength());
}
public ColumnInfo toProto(TiTableInfo table) {
return toProtoBuilder(table).build();
}
ColumnInfo.Builder toProtoBuilder(TiTableInfo table) {
return ColumnInfo.newBuilder()
.setColumnId(id)
.setTp(type.getTypeCode())
.setCollation(type.getCollationCode())
.setColumnLen((int) type.getLength())
.setDecimal(type.getDecimal())
.setFlag(type.getFlag())
.setDefaultVal(getOriginDefaultValueAsByteString())
.setPkHandle(table.isPkHandle() && isPrimaryKey())
.addAllElems(type.getElems());
}
@Override
public boolean equals(Object other) {
if (other == this) {
return true;
}
if (!(other instanceof TiColumnInfo)) {
return false;
}
TiColumnInfo col = (TiColumnInfo) other;
return Objects.equals(id, col.id)
&& Objects.equals(name, col.name)
&& Objects.equals(type, col.type)
&& Objects.equals(schemaState, col.schemaState)
&& isPrimaryKey == col.isPrimaryKey
&& Objects.equals(defaultValue, col.defaultValue)
&& Objects.equals(originDefaultValue, col.originDefaultValue);
}
@Override
public int hashCode() {
return Objects.hash(
id, name, type, schemaState, isPrimaryKey, defaultValue, originDefaultValue);
}
}

822
src/main/java/com/pingcap/tikv/meta/TiDAGRequest.java Normal file
View File

@ -0,0 +1,822 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.meta;
import static com.google.common.base.Preconditions.checkArgument;
import static com.pingcap.tikv.predicates.PredicateUtils.mergeCNFExpressions;
import static java.util.Objects.requireNonNull;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.pingcap.tidb.tipb.*;
import com.pingcap.tikv.codec.KeyUtils;
import com.pingcap.tikv.exception.DAGRequestException;
import com.pingcap.tikv.exception.TiClientInternalException;
import com.pingcap.tikv.expression.ByItem;
import com.pingcap.tikv.expression.ColumnRef;
import com.pingcap.tikv.expression.Expression;
import com.pingcap.tikv.expression.visitor.ExpressionTypeCoercer;
import com.pingcap.tikv.expression.visitor.MetaResolver;
import com.pingcap.tikv.expression.visitor.ProtoConverter;
import com.pingcap.tikv.key.RowKey;
import com.pingcap.tikv.kvproto.Coprocessor;
import com.pingcap.tikv.types.DataType;
import com.pingcap.tikv.util.KeyRangeUtils;
import com.pingcap.tikv.util.Pair;
import java.io.*;
import java.util.*;
import java.util.stream.Collectors;
/**
* Type TiDAGRequest.
*
* <p>Used for constructing a new DAG request to TiKV
*/
public class TiDAGRequest implements Serializable {
public static class Builder {
private List<String> requiredCols = new ArrayList<>();
private List<Expression> filters = new ArrayList<>();
private List<ByItem> orderBys = new ArrayList<>();
private List<Coprocessor.KeyRange> ranges = new ArrayList<>();
private TiTableInfo tableInfo;
private int limit;
private long startTs;
public static Builder newBuilder() {
return new Builder();
}
public Builder setFullTableScan(TiTableInfo tableInfo) {
requireNonNull(tableInfo);
setTableInfo(tableInfo);
RowKey start = RowKey.createMin(tableInfo.getId());
RowKey end = RowKey.createBeyondMax(tableInfo.getId());
ranges.add(KeyRangeUtils.makeCoprocRange(start.toByteString(), end.toByteString()));
return this;
}
public Builder setLimit(int limit) {
this.limit = limit;
return this;
}
public Builder setTableInfo(TiTableInfo tableInfo) {
this.tableInfo = tableInfo;
return this;
}
public Builder addRequiredCols(String... cols) {
this.requiredCols.addAll(Arrays.asList(cols));
return this;
}
public Builder addRequiredCols(List<String> cols) {
this.requiredCols.addAll(cols);
return this;
}
public Builder addFilter(Expression filter) {
this.filters.add(filter);
return this;
}
public Builder addOrderBy(ByItem item) {
this.orderBys.add(item);
return this;
}
public Builder setStartTs(long ts) {
this.startTs = ts;
return this;
}
public TiDAGRequest build(PushDownType pushDownType) {
TiDAGRequest req = new TiDAGRequest(pushDownType);
req.setTableInfo(tableInfo);
req.addRanges(ranges);
filters.forEach(req::addFilter);
if (!orderBys.isEmpty()) {
orderBys.forEach(req::addOrderByItem);
}
if (limit != 0) {
req.setLimit(limit);
}
requiredCols.forEach(c -> req.addRequiredColumn(ColumnRef.create(c)));
req.setStartTs(startTs);
req.resolve();
return req;
}
}
public TiDAGRequest(PushDownType pushDownType) {
this.pushDownType = pushDownType;
}
public TiDAGRequest(PushDownType pushDownType, int timeZoneOffset) {
this(pushDownType);
this.timeZoneOffset = timeZoneOffset;
}
public enum TruncateMode {
IgnoreTruncation(0x1),
TruncationAsWarning(0x2);
private final long mask;
TruncateMode(long mask) {
this.mask = mask;
}
public long mask(long flags) {
return flags | mask;
}
}
/** Whether we use streaming to push down the request */
public enum PushDownType {
STREAMING,
NORMAL
}
/** Predefined executor priority map. */
private static final Map<ExecType, Integer> EXEC_TYPE_PRIORITY_MAP =
ImmutableMap.<ExecType, Integer>builder()
.put(ExecType.TypeTableScan, 0)
.put(ExecType.TypeIndexScan, 0)
.put(ExecType.TypeSelection, 1)
.put(ExecType.TypeAggregation, 2)
.put(ExecType.TypeTopN, 3)
.put(ExecType.TypeLimit, 4)
.build();
private TiTableInfo tableInfo;
private TiIndexInfo indexInfo;
private final List<ColumnRef> fields = new ArrayList<>();
private final List<Expression> filters = new ArrayList<>();
private final List<ByItem> groupByItems = new ArrayList<>();
private final List<ByItem> orderByItems = new ArrayList<>();
private List<Expression> pushdownFilters = null;
// System like Spark has different type promotion rules
// we need a cast to target when given
private final List<Pair<Expression, DataType>> aggregates = new ArrayList<>();
private final List<Coprocessor.KeyRange> keyRanges = new ArrayList<>();
// If index scanning of this request is not possible in some scenario, we downgrade it to a table
// scan and use
// downGradeRanges instead of index scan ranges stored in keyRanges along with downgradeFilters to
// perform a
// table scan.
private List<Expression> downgradeFilters = new ArrayList<>();
private int limit;
private int timeZoneOffset;
private long flags;
private long startTs;
private Expression having;
private boolean distinct;
private boolean handleNeeded;
private boolean isDoubleRead;
private final PushDownType pushDownType;
private IdentityHashMap<Expression, DataType> typeMap;
private double estimatedCount = -1;
private static ColumnInfo handleColumn =
ColumnInfo.newBuilder()
.setColumnId(-1)
.setPkHandle(true)
// We haven't changed the field name in protobuf file, but
// we need to set this to true in order to retrieve the handle,
// so the name 'setPkHandle' may sounds strange.
.build();
private List<Expression> getAllExpressions() {
ImmutableList.Builder<Expression> builder = ImmutableList.builder();
builder.addAll(getFields());
builder.addAll(getFilters());
builder.addAll(getAggregates());
getGroupByItems().forEach(item -> builder.add(item.getExpr()));
getOrderByItems().forEach(item -> builder.add(item.getExpr()));
if (having != null) {
builder.add(having);
}
return builder.build();
}
public DataType getExpressionType(Expression expression) {
requireNonNull(typeMap, "request is not resolved");
return typeMap.get(expression);
}
public void resolve() {
MetaResolver resolver = new MetaResolver(tableInfo);
ExpressionTypeCoercer inferrer = new ExpressionTypeCoercer();
resolver.resolve(getAllExpressions());
inferrer.infer(getAllExpressions());
typeMap = inferrer.getTypeMap();
}
/**
* Unify indexScan and tableScan building logic since they are very much alike. DAGRequest for
* IndexScan should also contain filters and aggregation, so we can reuse this part of logic.
*
* <p>DAGRequest is made up of a chain of executors with strict orders: TableScan/IndexScan >
* Selection > Aggregation > TopN/Limit a DAGRequest must contain one and only one TableScan or
* IndexScan.
*
* @param isIndexScan whether the dagRequest to build is an IndexScan
* @return final DAGRequest built
*/
public DAGRequest buildScan(boolean isIndexScan) {
checkArgument(startTs != 0, "timestamp is 0");
DAGRequest.Builder dagRequestBuilder = DAGRequest.newBuilder();
Executor.Builder executorBuilder = Executor.newBuilder();
IndexScan.Builder indexScanBuilder = IndexScan.newBuilder();
TableScan.Builder tblScanBuilder = TableScan.newBuilder();
// find a column's offset in fields
Map<ColumnRef, Integer> colOffsetInFieldMap = new HashMap<>();
// find a column's position in index
Map<TiColumnInfo, Integer> colPosInIndexMap = new HashMap<>();
if (isIndexScan) {
// IndexScan
if (indexInfo == null) {
throw new TiClientInternalException("Index is empty for index scan");
}
List<TiColumnInfo> columnInfoList = tableInfo.getColumns();
boolean hasPk = false;
// We extract index column info
List<Integer> indexColOffsets =
indexInfo
.getIndexColumns()
.stream()
.map(TiIndexColumn::getOffset)
.collect(Collectors.toList());
int idxPos = 0;
// for index scan builder, columns are added by its order in index
for (Integer idx : indexColOffsets) {
TiColumnInfo tiColumnInfo = columnInfoList.get(idx);
ColumnInfo columnInfo = tiColumnInfo.toProto(tableInfo);
colPosInIndexMap.put(tiColumnInfo, idxPos++);
ColumnInfo.Builder colBuilder = ColumnInfo.newBuilder(columnInfo);
if (columnInfo.getColumnId() == -1) {
hasPk = true;
colBuilder.setPkHandle(true);
}
indexScanBuilder.addColumns(colBuilder);
}
if (isDoubleRead()) {
// double read case
if (!hasPk) {
indexScanBuilder.addColumns(handleColumn);
}
int colCount = indexScanBuilder.getColumnsCount();
// double read case: need to retrieve handle
dagRequestBuilder.addOutputOffsets(colCount != 0 ? colCount - 1 : 0);
} else {
int colCount = indexScanBuilder.getColumnsCount();
boolean pkIsNeeded = false;
// =================== IMPORTANT ======================
// offset for dagRequest should be in accordance with fields
for (ColumnRef col : getFields()) {
Integer pos = colPosInIndexMap.get(col.getColumnInfo());
if (pos != null) {
TiColumnInfo columnInfo = columnInfoList.get(indexColOffsets.get(pos));
if (col.getColumnInfo().equals(columnInfo)) {
dagRequestBuilder.addOutputOffsets(pos);
colOffsetInFieldMap.put(col, pos);
}
}
// if a column of field is not contained in index selected,
// logically it must be the pk column and
// the pkIsHandle must be true. Extra check here.
else if (col.getColumnInfo().isPrimaryKey() && tableInfo.isPkHandle()) {
pkIsNeeded = true;
// offset should be processed for each primary key encountered
dagRequestBuilder.addOutputOffsets(colCount);
// for index scan, column offset must be in the order of index->handle
colOffsetInFieldMap.put(col, indexColOffsets.size());
} else {
throw new DAGRequestException(
"columns other than primary key and index key exist in fields while index single read: "
+ col.getName());
}
}
// pk is not included in index but still needed
if (pkIsNeeded) {
indexScanBuilder.addColumns(handleColumn);
}
}
executorBuilder.setTp(ExecType.TypeIndexScan);
indexScanBuilder.setTableId(tableInfo.getId()).setIndexId(indexInfo.getId());
dagRequestBuilder.addExecutors(executorBuilder.setIdxScan(indexScanBuilder).build());
} else {
// TableScan
executorBuilder.setTp(ExecType.TypeTableScan);
tblScanBuilder.setTableId(tableInfo.getId());
// Step1. Add columns to first executor
for (int i = 0; i < getFields().size(); i++) {
ColumnRef col = getFields().get(i);
tblScanBuilder.addColumns(col.getColumnInfo().toProto(tableInfo));
colOffsetInFieldMap.put(col, i);
}
// Currently, according to TiKV's implementation, if handle
// is needed, we should add an extra column with an ID of -1
// to the TableScan executor
if (isHandleNeeded()) {
tblScanBuilder.addColumns(handleColumn);
}
dagRequestBuilder.addExecutors(executorBuilder.setTblScan(tblScanBuilder));
// column offset should be in accordance with fields
for (int i = 0; i < getFields().size(); i++) {
dagRequestBuilder.addOutputOffsets(i);
}
// if handle is needed, we should append one output offset
if (isHandleNeeded()) {
dagRequestBuilder.addOutputOffsets(tableInfo.getColumns().size());
}
}
if (!isIndexScan || (isIndexScan() && !isDoubleRead())) {
// clear executorBuilder
executorBuilder.clear();
// Step2. Add others
// DO NOT EDIT EXPRESSION CONSTRUCTION ORDER
// Or make sure the construction order is below:
// TableScan/IndexScan > Selection > Aggregation > TopN/Limit
Expression whereExpr = mergeCNFExpressions(getFilters());
if (whereExpr != null) {
executorBuilder.setTp(ExecType.TypeSelection);
dagRequestBuilder.addExecutors(
executorBuilder.setSelection(
Selection.newBuilder()
.addConditions(ProtoConverter.toProto(whereExpr, colOffsetInFieldMap))));
executorBuilder.clear();
}
if (!getGroupByItems().isEmpty() || !getAggregates().isEmpty()) {
Aggregation.Builder aggregationBuilder = Aggregation.newBuilder();
getGroupByItems()
.forEach(
tiByItem ->
aggregationBuilder.addGroupBy(
ProtoConverter.toProto(tiByItem.getExpr(), colOffsetInFieldMap)));
getAggregates()
.forEach(
tiExpr ->
aggregationBuilder.addAggFunc(
ProtoConverter.toProto(tiExpr, colOffsetInFieldMap)));
executorBuilder.setTp(ExecType.TypeAggregation);
dagRequestBuilder.addExecutors(executorBuilder.setAggregation(aggregationBuilder));
executorBuilder.clear();
}
if (!getOrderByItems().isEmpty()) {
TopN.Builder topNBuilder = TopN.newBuilder();
getOrderByItems()
.forEach(
tiByItem ->
topNBuilder.addOrderBy(
com.pingcap.tidb.tipb.ByItem.newBuilder()
.setExpr(
ProtoConverter.toProto(tiByItem.getExpr(), colOffsetInFieldMap))
.setDesc(tiByItem.isDesc())));
executorBuilder.setTp(ExecType.TypeTopN);
topNBuilder.setLimit(getLimit());
dagRequestBuilder.addExecutors(executorBuilder.setTopN(topNBuilder));
executorBuilder.clear();
} else if (getLimit() != 0) {
Limit.Builder limitBuilder = Limit.newBuilder();
limitBuilder.setLimit(getLimit());
executorBuilder.setTp(ExecType.TypeLimit);
dagRequestBuilder.addExecutors(executorBuilder.setLimit(limitBuilder));
executorBuilder.clear();
}
}
DAGRequest request =
dagRequestBuilder
.setTimeZoneOffset(timeZoneOffset)
.setFlags(flags)
.setStartTs(startTs)
.build();
validateRequest(request);
return request;
}
/**
* Check if a DAG request is valid.
*
* <p>Note: When constructing a DAG request, a executor with an ExecType of higher priority should
* always be placed before those lower ones.
*
* @param dagRequest Request DAG.
*/
private void validateRequest(DAGRequest dagRequest) {
requireNonNull(dagRequest);
// A DAG request must has at least one executor.
if (dagRequest.getExecutorsCount() < 1) {
throw new DAGRequestException("Invalid executors count:" + dagRequest.getExecutorsCount());
}
ExecType formerType = dagRequest.getExecutors(0).getTp();
if (formerType != ExecType.TypeTableScan && formerType != ExecType.TypeIndexScan) {
throw new DAGRequestException(
"Invalid first executor type:"
+ formerType
+ ", must one of TypeTableScan or TypeIndexScan");
}
for (int i = 1; i < dagRequest.getExecutorsCount(); i++) {
ExecType currentType = dagRequest.getExecutors(i).getTp();
if (EXEC_TYPE_PRIORITY_MAP.get(currentType) < EXEC_TYPE_PRIORITY_MAP.get(formerType)) {
throw new DAGRequestException("Invalid executor priority.");
}
formerType = currentType;
}
}
public TiDAGRequest setTableInfo(TiTableInfo tableInfo) {
this.tableInfo = requireNonNull(tableInfo, "tableInfo is null");
return this;
}
public TiTableInfo getTableInfo() {
return this.tableInfo;
}
public TiDAGRequest setIndexInfo(TiIndexInfo indexInfo) {
this.indexInfo = requireNonNull(indexInfo, "indexInfo is null");
return this;
}
public TiIndexInfo getIndexInfo() {
return indexInfo;
}
public void clearIndexInfo() {
indexInfo = null;
}
public int getLimit() {
return limit;
}
/**
* add limit clause to select query.
*
* @param limit is just a integer.
* @return a SelectBuilder
*/
public TiDAGRequest setLimit(int limit) {
this.limit = limit;
return this;
}
/**
* set timezone offset
*
* @param timeZoneOffset timezone offset
* @return a TiDAGRequest
*/
public TiDAGRequest setTimeZoneOffset(int timeZoneOffset) {
this.timeZoneOffset = timeZoneOffset;
return this;
}
int getTimeZoneOffset() {
return timeZoneOffset;
}
/**
* set truncate mode
*
* @param mode truncate mode
* @return a TiDAGRequest
*/
public TiDAGRequest setTruncateMode(TiDAGRequest.TruncateMode mode) {
flags = requireNonNull(mode, "mode is null").mask(flags);
return this;
}
@VisibleForTesting
public long getFlags() {
return flags;
}
/**
* set start timestamp for the transaction
*
* @param startTs timestamp
* @return a TiDAGRequest
*/
public TiDAGRequest setStartTs(long startTs) {
this.startTs = startTs;
return this;
}
long getStartTs() {
return startTs;
}
/**
* set having clause to select query
*
* @param having is a expression represents Having
* @return a TiDAGRequest
*/
public TiDAGRequest setHaving(Expression having) {
this.having = requireNonNull(having, "having is null");
return this;
}
public TiDAGRequest setDistinct(boolean distinct) {
this.distinct = distinct;
return this;
}
public boolean isDistinct() {
return distinct;
}
public TiDAGRequest addAggregate(Expression expr, DataType targetType) {
requireNonNull(expr, "aggregation expr is null");
aggregates.add(Pair.create(expr, targetType));
return this;
}
public List<Expression> getAggregates() {
return aggregates.stream().map(p -> p.first).collect(Collectors.toList());
}
public List<Pair<Expression, DataType>> getAggregatePairs() {
return aggregates;
}
/**
* add a order by clause to select query.
*
* @param byItem is a TiByItem.
* @return a SelectBuilder
*/
public TiDAGRequest addOrderByItem(ByItem byItem) {
orderByItems.add(requireNonNull(byItem, "byItem is null"));
return this;
}
List<ByItem> getOrderByItems() {
return orderByItems;
}
/**
* add a group by clause to select query
*
* @param byItem is a TiByItem
* @return a SelectBuilder
*/
public TiDAGRequest addGroupByItem(ByItem byItem) {
groupByItems.add(requireNonNull(byItem, "byItem is null"));
return this;
}
public List<ByItem> getGroupByItems() {
return groupByItems;
}
/**
* Field is not support in TiDB yet, for here we simply allow TiColumnRef instead of TiExpr like
* in SelectRequest proto
*
* <p>
*
* <p>This interface allows duplicate columns and it's user's responsibility to do dedup since we
* need to ensure exact order and items preserved during decoding
*
* @param column is column referred during selectReq
*/
public TiDAGRequest addRequiredColumn(ColumnRef column) {
fields.add(requireNonNull(column, "columnRef is null"));
return this;
}
public List<ColumnRef> getFields() {
return fields;
}
/**
* set key range of scan
*
* @param ranges key range of scan
*/
public TiDAGRequest addRanges(List<Coprocessor.KeyRange> ranges) {
keyRanges.addAll(requireNonNull(ranges, "KeyRange is null"));
return this;
}
public void resetFilters(List<Expression> filters) {
this.filters.clear();
this.filters.addAll(filters);
}
public List<Coprocessor.KeyRange> getRanges() {
return keyRanges;
}
public TiDAGRequest addFilter(Expression filter) {
this.filters.add(requireNonNull(filter, "filters expr is null"));
return this;
}
public List<Expression> getDowngradeFilters() {
return downgradeFilters;
}
public TiDAGRequest addDowngradeFilter(Expression filter) {
this.downgradeFilters.add(requireNonNull(filter, "downgrade filter is null"));
return this;
}
/**
* Check whether the DAG request has any aggregate expression.
*
* @return the boolean
*/
public boolean hasAggregate() {
return !getAggregates().isEmpty();
}
/**
* Check whether the DAG request has any group by expression.
*
* @return the boolean
*/
public boolean hasGroupBy() {
return !getGroupByItems().isEmpty();
}
public List<Expression> getFilters() {
return filters;
}
/**
* Returns whether handle is needed.
*
* @return the boolean
*/
public boolean isHandleNeeded() {
return handleNeeded;
}
/**
* Sets handle needed.
*
* @param handleNeeded the handle needed
*/
public void setHandleNeeded(boolean handleNeeded) {
this.handleNeeded = handleNeeded;
}
/**
* Returns whether needs double read
*
* @return boolean
*/
public boolean isDoubleRead() {
return isDoubleRead;
}
/**
* Sets isDoubleRead
*
* @param isDoubleRead if is double read
*/
public void setIsDoubleRead(boolean isDoubleRead) {
this.isDoubleRead = isDoubleRead;
}
/**
* Returns whether this request is of indexScanType
*
* @return true iff indexInfo is provided, false otherwise
*/
public boolean isIndexScan() {
return indexInfo != null;
}
/**
* Whether we use streaming processing to retrieve data
*
* @return push down type.
*/
public PushDownType getPushDownType() {
return pushDownType;
}
/** Set the estimated row count will be fetched from this request. */
public void setEstimatedCount(double estimatedCount) {
this.estimatedCount = estimatedCount;
}
/** Get the estimated row count will be fetched from this request. */
public double getEstimatedCount() {
return estimatedCount;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
if (tableInfo != null) {
sb.append(String.format("[table: %s] ", tableInfo.getName()));
}
if (indexInfo != null) {
sb.append(String.format("[Index: %s] ", indexInfo.getName()));
}
if (!getFields().isEmpty()) {
sb.append(", Columns: ");
Joiner.on(", ").skipNulls().appendTo(sb, getFields());
}
if (!getDowngradeFilters().isEmpty()) {
// should be called after all parameters are set
if (pushdownFilters == null) {
pushdownFilters = new ArrayList<>(getDowngradeFilters());
pushdownFilters.removeAll(new HashSet<>(getFilters()));
}
if (!pushdownFilters.isEmpty()) {
sb.append(", Pushdown Filter: ");
Joiner.on(", ").skipNulls().appendTo(sb, pushdownFilters);
}
}
if (!getFilters().isEmpty()) {
sb.append(", Residual Filter: ");
Joiner.on(", ").skipNulls().appendTo(sb, getFilters());
}
// Key ranges might be also useful
if (!getRanges().isEmpty()) {
sb.append(", KeyRange: ");
getRanges().forEach(x -> sb.append(KeyUtils.formatBytes(x)));
}
if (!getAggregates().isEmpty()) {
sb.append(", Aggregates: ");
Joiner.on(", ").skipNulls().appendTo(sb, getAggregates());
}
if (!getGroupByItems().isEmpty()) {
sb.append(", Group By: ");
Joiner.on(", ").skipNulls().appendTo(sb, getGroupByItems());
}
if (!getOrderByItems().isEmpty()) {
sb.append(", Order By: ");
Joiner.on(", ").skipNulls().appendTo(sb, getOrderByItems());
}
if (getLimit() != 0) {
sb.append(", Limit: ");
sb.append("[").append(limit).append("]");
}
return sb.toString();
}
public TiDAGRequest copy() {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ObjectOutputStream oos = new ObjectOutputStream(baos);
oos.writeObject(this);
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
ObjectInputStream ois = new ObjectInputStream(bais);
return ((TiDAGRequest) ois.readObject());
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}

109
src/main/java/com/pingcap/tikv/meta/TiDBInfo.java Normal file
View File

@ -0,0 +1,109 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.meta;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;
@JsonIgnoreProperties(ignoreUnknown = true)
public class TiDBInfo {
private final long id;
private final String name;
private final String charset;
private final String collate;
private final List<TiTableInfo> tables;
private final SchemaState schemaState;
@JsonCreator
public TiDBInfo(
@JsonProperty("id") long id,
@JsonProperty("db_name") CIStr name,
@JsonProperty("charset") String charset,
@JsonProperty("collate") String collate,
@JsonProperty("-") List<TiTableInfo> tables,
@JsonProperty("state") int schemaState) {
this.id = id;
this.name = name.getL();
this.charset = charset;
this.collate = collate;
this.tables = tables;
this.schemaState = SchemaState.fromValue(schemaState);
}
private TiDBInfo(
long id,
String name,
String charset,
String collate,
List<TiTableInfo> tables,
SchemaState schemaState) {
this.id = id;
this.name = name;
this.charset = charset;
this.collate = collate;
this.tables = tables;
this.schemaState = schemaState;
}
public TiDBInfo rename(String newName) {
return new TiDBInfo(id, newName, charset, collate, tables, schemaState);
}
public long getId() {
return id;
}
public String getName() {
return name;
}
public String getCharset() {
return charset;
}
public String getCollate() {
return collate;
}
public List<TiTableInfo> getTables() {
return tables;
}
SchemaState getSchemaState() {
return schemaState;
}
@Override
public boolean equals(Object other) {
if (other == this) {
return true;
}
if (!(other instanceof TiDBInfo)) {
return false;
}
TiDBInfo otherDB = (TiDBInfo) other;
return otherDB.getId() == getId() && otherDB.getName().equals(getName());
}
@Override
public int hashCode() {
final int prime = 31;
int result = prime + Long.hashCode(getId());
return result * prime + getName().hashCode();
}
}

65
src/main/java/com/pingcap/tikv/meta/TiIndexColumn.java Normal file
View File

@ -0,0 +1,65 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.meta;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.pingcap.tikv.types.DataType;
import java.io.Serializable;
@JsonIgnoreProperties(ignoreUnknown = true)
public class TiIndexColumn implements Serializable {
private String name;
private int offset;
private long length;
@JsonCreator
public TiIndexColumn(
@JsonProperty("name") CIStr name,
@JsonProperty("offset") int offset,
@JsonProperty("length") long length) {
this.name = name.getL();
this.offset = offset;
this.length = length;
}
public String getName() {
return name;
}
public int getOffset() {
return offset;
}
public long getLength() {
return length;
}
public boolean isPrefixIndex() {
return length != DataType.UNSPECIFIED_LEN;
}
public boolean matchName(String otherName) {
return name.equalsIgnoreCase(otherName);
}
@Override
public String toString() {
return String.format(
"%s {name: %s, offset: %d, length: %d}", getClass().getSimpleName(), name, offset, length);
}
}

167
src/main/java/com/pingcap/tikv/meta/TiIndexInfo.java Normal file
View File

@ -0,0 +1,167 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.meta;
import static java.util.Objects.requireNonNull;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.pingcap.tidb.tipb.ColumnInfo;
import com.pingcap.tidb.tipb.IndexInfo;
import java.io.Serializable;
import java.util.List;
import java.util.stream.Collectors;
@JsonIgnoreProperties(ignoreUnknown = true)
public class TiIndexInfo implements Serializable {
private final long id;
private final String name;
private final String tableName;
private final List<TiIndexColumn> indexColumns;
private final boolean isUnique;
private final boolean isPrimary;
private final SchemaState schemaState;
private final String comment;
private final IndexType indexType;
private final boolean isFakePrimaryKey;
@JsonCreator
@VisibleForTesting
public TiIndexInfo(
@JsonProperty("id") long id,
@JsonProperty("idx_name") CIStr name,
@JsonProperty("tbl_name") CIStr tableName,
@JsonProperty("idx_cols") List<TiIndexColumn> indexColumns,
@JsonProperty("is_unique") boolean isUnique,
@JsonProperty("is_primary") boolean isPrimary,
@JsonProperty("state") int schemaState,
@JsonProperty("comment") String comment,
@JsonProperty("index_type") int indexType,
// This is a fake property and added JsonProperty only to
// to bypass Jackson frameworks's check
@JsonProperty("___isFakePrimaryKey") boolean isFakePrimaryKey) {
this.id = id;
this.name = requireNonNull(name, "index name is null").getL();
this.tableName = requireNonNull(tableName, "table name is null").getL();
this.indexColumns = ImmutableList.copyOf(requireNonNull(indexColumns, "indexColumns is null"));
this.isUnique = isUnique;
this.isPrimary = isPrimary;
this.schemaState = SchemaState.fromValue(schemaState);
this.comment = comment;
this.indexType = IndexType.fromValue(indexType);
this.isFakePrimaryKey = isFakePrimaryKey;
}
public static TiIndexInfo generateFakePrimaryKeyIndex(TiTableInfo table) {
TiColumnInfo pkColumn = table.getPrimaryKeyColumn();
if (pkColumn != null) {
return new TiIndexInfo(
-1,
CIStr.newCIStr("fake_pk_" + table.getId()),
CIStr.newCIStr(table.getName()),
ImmutableList.of(pkColumn.toFakeIndexColumn()),
true,
true,
SchemaState.StatePublic.getStateCode(),
"Fake Column",
IndexType.IndexTypeHash.getTypeCode(),
true);
}
return null;
}
public long getId() {
return id;
}
public String getName() {
return name;
}
public String getTableName() {
return tableName;
}
public List<TiIndexColumn> getIndexColumns() {
return indexColumns;
}
public boolean isUnique() {
return isUnique;
}
public boolean isPrimary() {
return isPrimary;
}
public SchemaState getSchemaState() {
return schemaState;
}
public String getComment() {
return comment;
}
public IndexType getIndexType() {
return indexType;
}
public IndexInfo toProto(TiTableInfo tableInfo) {
IndexInfo.Builder builder =
IndexInfo.newBuilder().setTableId(tableInfo.getId()).setIndexId(id).setUnique(isUnique);
List<TiColumnInfo> columns = tableInfo.getColumns();
for (TiIndexColumn indexColumn : getIndexColumns()) {
int offset = indexColumn.getOffset();
TiColumnInfo column = columns.get(offset);
builder.addColumns(column.toProto(tableInfo));
}
if (tableInfo.isPkHandle()) {
for (TiColumnInfo column : columns) {
if (!column.isPrimaryKey()) {
continue;
}
ColumnInfo pbColumn = column.toProtoBuilder(tableInfo).setPkHandle(true).build();
builder.addColumns(pbColumn);
}
}
return builder.build();
}
public boolean isFakePrimaryKey() {
return isFakePrimaryKey;
}
@Override
public String toString() {
return String.format(
"%s[%s]",
name,
Joiner.on(",")
.skipNulls()
.join(
indexColumns
.stream()
.map(column -> column.getName())
.collect(Collectors.toList())));
}
}

28
src/main/java/com/pingcap/tikv/meta/TiPartitionDef.java Normal file
View File

@ -0,0 +1,28 @@
package com.pingcap.tikv.meta;
import com.google.common.annotations.VisibleForTesting;
import java.io.Serializable;
import org.codehaus.jackson.annotate.JsonCreator;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
@JsonIgnoreProperties(ignoreUnknown = true)
public class TiPartitionDef implements Serializable {
private final long id;
private final CIStr name;
private final String[] lessThan;
private final String comment;
@JsonCreator
@VisibleForTesting
public TiPartitionDef(
@JsonProperty("id") long id,
@JsonProperty("name") CIStr name,
@JsonProperty("less_than") String[] lessThan,
@JsonProperty("comment") String comment) {
this.id = id;
this.name = name;
this.lessThan = lessThan;
this.comment = comment;
}
}

36
src/main/java/com/pingcap/tikv/meta/TiPartitionInfo.java Normal file
View File

@ -0,0 +1,36 @@
package com.pingcap.tikv.meta;
import com.google.common.annotations.VisibleForTesting;
import org.codehaus.jackson.annotate.JsonCreator;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
@JsonIgnoreProperties(ignoreUnknown = true)
public class TiPartitionInfo {
public static enum PartitionType {
RangePartition,
HashPartition,
ListPartition,
}
private final PartitionType type;
private final String expr;
private final CIStr[] columns;
private final boolean enable;
private final TiPartitionDef[] defs;
@JsonCreator
@VisibleForTesting
public TiPartitionInfo(
@JsonProperty("type") PartitionType type,
@JsonProperty("expr") String expr,
@JsonProperty("columns") CIStr[] columns,
@JsonProperty("enable") boolean enable,
@JsonProperty("definitions") TiPartitionDef[] defs) {
this.type = type;
this.expr = expr;
this.columns = columns;
this.enable = enable;
this.defs = defs;
}
}

199
src/main/java/com/pingcap/tikv/meta/TiTableInfo.java Normal file
View File

@ -0,0 +1,199 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.meta;
import static java.util.Objects.requireNonNull;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableList;
import com.pingcap.tidb.tipb.TableInfo;
import com.pingcap.tikv.exception.TiClientInternalException;
import com.pingcap.tikv.meta.TiColumnInfo.InternalTypeHolder;
import com.pingcap.tikv.types.DataType;
import com.pingcap.tikv.types.DataTypeFactory;
import java.io.Serializable;
import java.util.List;
import java.util.stream.Collectors;
@JsonIgnoreProperties(ignoreUnknown = true)
public class TiTableInfo implements Serializable {
private final long id;
private final String name;
private final String charset;
private final String collate;
private final List<TiColumnInfo> columns;
private final List<TiIndexInfo> indices;
private final boolean pkIsHandle;
private final String comment;
private final long autoIncId;
private final long maxColumnId;
private final long maxIndexId;
private final long oldSchemaId;
private final TiPartitionInfo partitionInfo;
@JsonCreator
public TiTableInfo(
@JsonProperty("id") long id,
@JsonProperty("name") CIStr name,
@JsonProperty("charset") String charset,
@JsonProperty("collate") String collate,
@JsonProperty("pk_is_handle") boolean pkIsHandle,
@JsonProperty("cols") List<TiColumnInfo> columns,
@JsonProperty("index_info") List<TiIndexInfo> indices,
@JsonProperty("comment") String comment,
@JsonProperty("auto_inc_id") long autoIncId,
@JsonProperty("max_col_id") long maxColumnId,
@JsonProperty("max_idx_id") long maxIndexId,
@JsonProperty("old_schema_id") long oldSchemaId,
@JsonProperty("partition") TiPartitionInfo partitionInfo) {
this.id = id;
this.name = name.getL();
this.charset = charset;
this.collate = collate;
this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null"));
this.pkIsHandle = pkIsHandle;
this.indices = indices != null ? ImmutableList.copyOf(indices) : ImmutableList.of();
this.comment = comment;
this.autoIncId = autoIncId;
this.maxColumnId = maxColumnId;
this.maxIndexId = maxIndexId;
this.oldSchemaId = oldSchemaId;
this.partitionInfo = partitionInfo;
}
public long getId() {
return id;
}
public String getName() {
return name;
}
public String getCharset() {
return charset;
}
public String getCollate() {
return collate;
}
public List<TiColumnInfo> getColumns() {
return columns;
}
public TiColumnInfo getColumn(int offset) {
if (offset < 0 || offset >= columns.size()) {
throw new TiClientInternalException(String.format("Column offset %d out of bound", offset));
}
return columns.get(offset);
}
public boolean isPkHandle() {
return pkIsHandle;
}
public List<TiIndexInfo> getIndices() {
return indices;
}
public String getComment() {
return comment;
}
public long getAutoIncId() {
return autoIncId;
}
public long getMaxColumnId() {
return maxColumnId;
}
public long getMaxIndexId() {
return maxIndexId;
}
public long getOldSchemaId() {
return oldSchemaId;
}
public TableInfo toProto() {
return TableInfo.newBuilder()
.setTableId(getId())
.addAllColumns(
getColumns().stream().map(col -> col.toProto(this)).collect(Collectors.toList()))
.build();
}
// Only Integer Column will be a PK column
// and there exists only one PK column
public TiColumnInfo getPrimaryKeyColumn() {
if (isPkHandle()) {
for (TiColumnInfo col : getColumns()) {
if (col.isPrimaryKey()) {
return col;
}
}
}
return null;
}
public TiTableInfo copyTableWithRowId() {
if (!isPkHandle()) {
ImmutableList.Builder<TiColumnInfo> newColumns = ImmutableList.builder();
for (TiColumnInfo col : getColumns()) {
DataType type = col.getType();
InternalTypeHolder typeHolder = type.toTypeHolder();
typeHolder.setFlag(type.getFlag() & (~DataType.PriKeyFlag));
DataType newType = DataTypeFactory.of(typeHolder);
TiColumnInfo newCol =
new TiColumnInfo(
col.getId(),
col.getName(),
col.getOffset(),
newType,
col.getSchemaState(),
col.getOriginDefaultValue(),
col.getDefaultValue(),
col.getComment());
newColumns.add(newCol.copyWithoutPrimaryKey());
}
newColumns.add(TiColumnInfo.getRowIdColumn(getColumns().size()));
return new TiTableInfo(
getId(),
CIStr.newCIStr(getName()),
getCharset(),
getCollate(),
true,
newColumns.build(),
getIndices(),
getComment(),
getAutoIncId(),
getMaxColumnId(),
getMaxIndexId(),
getOldSchemaId(),
partitionInfo);
} else {
return this;
}
}
@Override
public String toString() {
return toProto().toString();
}
}

43
src/main/java/com/pingcap/tikv/meta/TiTimestamp.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.meta;
import java.io.Serializable;
/** TiTimestamp is the timestamp returned by timestamp oracle inside placement driver */
public class TiTimestamp implements Serializable {
private static final int PHYSICAL_SHIFT_BITS = 18;
private final long physical;
private final long logical;
public TiTimestamp(long p, long l) {
this.physical = p;
this.logical = l;
}
public long getVersion() {
return (physical << PHYSICAL_SHIFT_BITS) + logical;
}
public long getPhysical() {
return this.physical;
}
public long getLogical() {
return this.logical;
}
}

40
src/main/java/com/pingcap/tikv/operation/ErrorHandler.java Normal file
View File

@ -0,0 +1,40 @@
/*
*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.pingcap.tikv.operation;
import com.pingcap.tikv.util.BackOffer;
public interface ErrorHandler<RespT> {
/**
* Handle the error received in the response after a calling process completes.
*
* @param backOffer Back offer used for retry
* @param resp the response to handle
* @return whether the caller should retry
*/
boolean handleResponseError(BackOffer backOffer, RespT resp);
/**
* Handle the error received during a calling process before it could return a normal response.
*
* @param backOffer Back offer used for retry
* @param e Exception received during a calling process
* @return whether the caller should retry
*/
boolean handleRequestError(BackOffer backOffer, Exception e);
}

255
src/main/java/com/pingcap/tikv/operation/KVErrorHandler.java Normal file
View File

@ -0,0 +1,255 @@
/*
*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.pingcap.tikv.operation;
import com.google.protobuf.ByteString;
import com.pingcap.tikv.codec.KeyUtils;
import com.pingcap.tikv.event.CacheInvalidateEvent;
import com.pingcap.tikv.exception.GrpcException;
import com.pingcap.tikv.kvproto.Errorpb;
import com.pingcap.tikv.region.RegionErrorReceiver;
import com.pingcap.tikv.region.RegionManager;
import com.pingcap.tikv.region.TiRegion;
import com.pingcap.tikv.util.BackOffFunction;
import com.pingcap.tikv.util.BackOffer;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
import java.util.function.Function;
import org.apache.log4j.Logger;
// TODO: consider refactor to Builder mode
public class KVErrorHandler<RespT> implements ErrorHandler<RespT> {
private static final Logger logger = Logger.getLogger(KVErrorHandler.class);
private static final int NO_LEADER_STORE_ID =
0; // if there's currently no leader of a store, store id is set to 0
private final Function<RespT, Errorpb.Error> getRegionError;
private final Function<CacheInvalidateEvent, Void> cacheInvalidateCallBack;
private final RegionManager regionManager;
private final RegionErrorReceiver recv;
private final TiRegion ctxRegion;
public KVErrorHandler(
RegionManager regionManager,
RegionErrorReceiver recv,
TiRegion ctxRegion,
Function<RespT, Errorpb.Error> getRegionError) {
this.ctxRegion = ctxRegion;
this.recv = recv;
this.regionManager = regionManager;
this.getRegionError = getRegionError;
this.cacheInvalidateCallBack =
regionManager != null && regionManager.getSession() != null
? regionManager.getSession().getCacheInvalidateCallback()
: null;
}
private Errorpb.Error getRegionError(RespT resp) {
if (getRegionError != null) {
return getRegionError.apply(resp);
}
return null;
}
private void invalidateRegionStoreCache(TiRegion ctxRegion) {
regionManager.invalidateRegion(ctxRegion.getId());
regionManager.invalidateStore(ctxRegion.getLeader().getStoreId());
notifyRegionStoreCacheInvalidate(
ctxRegion.getId(),
ctxRegion.getLeader().getStoreId(),
CacheInvalidateEvent.CacheType.REGION_STORE);
}
/** Used for notifying Spark driver to invalidate cache from Spark workers. */
private void notifyRegionStoreCacheInvalidate(
long regionId, long storeId, CacheInvalidateEvent.CacheType type) {
if (cacheInvalidateCallBack != null) {
cacheInvalidateCallBack.apply(new CacheInvalidateEvent(regionId, storeId, true, true, type));
logger.info(
"Accumulating cache invalidation info to driver:regionId="
+ regionId
+ ",storeId="
+ storeId
+ ",type="
+ type.name());
} else {
logger.warn(
"Failed to send notification back to driver since CacheInvalidateCallBack is null in executor node.");
}
}
private void notifyRegionCacheInvalidate(long regionId) {
if (cacheInvalidateCallBack != null) {
cacheInvalidateCallBack.apply(
new CacheInvalidateEvent(
regionId, 0, true, false, CacheInvalidateEvent.CacheType.REGION_STORE));
logger.info(
"Accumulating cache invalidation info to driver:regionId="
+ regionId
+ ",type="
+ CacheInvalidateEvent.CacheType.REGION_STORE.name());
} else {
logger.warn(
"Failed to send notification back to driver since CacheInvalidateCallBack is null in executor node.");
}
}
private void notifyStoreCacheInvalidate(long storeId) {
if (cacheInvalidateCallBack != null) {
cacheInvalidateCallBack.apply(
new CacheInvalidateEvent(
0, storeId, false, true, CacheInvalidateEvent.CacheType.REGION_STORE));
} else {
logger.warn(
"Failed to send notification back to driver since CacheInvalidateCallBack is null in executor node.");
}
}
// Referenced from TiDB
// store/tikv/region_request.go - onRegionError
@Override
public boolean handleResponseError(BackOffer backOffer, RespT resp) {
if (resp == null) {
String msg =
String.format("Request Failed with unknown reason for region region [%s]", ctxRegion);
logger.warn(msg);
return handleRequestError(backOffer, new GrpcException(msg));
}
// Region error handling logic
Errorpb.Error error = getRegionError(resp);
if (error != null) {
if (error.hasNotLeader()) {
// this error is reported from raftstore:
// peer of current request is not leader, the following might be its causes:
// 1. cache is outdated, region has changed its leader, can be solved by re-fetching from PD
// 2. leader of current region is missing, need to wait and then fetch region info from PD
long newStoreId = error.getNotLeader().getLeader().getStoreId();
boolean retry = true;
// update Leader here
logger.warn(
String.format(
"NotLeader Error with region id %d and store id %d, new store id %d",
ctxRegion.getId(), ctxRegion.getLeader().getStoreId(), newStoreId));
BackOffFunction.BackOffFuncType backOffFuncType;
// if there's current no leader, we do not trigger update pd cache logic
// since issuing store = NO_LEADER_STORE_ID requests to pd will definitely fail.
if (newStoreId != NO_LEADER_STORE_ID) {
if (!this.regionManager.updateLeader(ctxRegion.getId(), newStoreId)
|| !recv.onNotLeader(this.regionManager.getStoreById(newStoreId))) {
// If update leader fails, we need to fetch new region info from pd,
// and re-split key range for new region. Setting retry to false will
// stop retry and enter handleCopResponse logic, which would use RegionMiss
// backOff strategy to wait, fetch new region and re-split key range.
// onNotLeader is only needed when updateLeader succeeds, thus switch
// to a new store address.
retry = false;
}
notifyRegionStoreCacheInvalidate(
ctxRegion.getId(), newStoreId, CacheInvalidateEvent.CacheType.LEADER);
backOffFuncType = BackOffFunction.BackOffFuncType.BoUpdateLeader;
} else {
logger.info(
String.format(
"Received zero store id, from region %d try next time", ctxRegion.getId()));
backOffFuncType = BackOffFunction.BackOffFuncType.BoRegionMiss;
}
backOffer.doBackOff(backOffFuncType, new GrpcException(error.toString()));
return retry;
} else if (error.hasStoreNotMatch()) {
// this error is reported from raftstore:
// store_id requested at the moment is inconsistent with that expected
// Solution：re-fetch from PD
long storeId = ctxRegion.getLeader().getStoreId();
logger.warn(
String.format(
"Store Not Match happened with region id %d, store id %d",
ctxRegion.getId(), storeId));
this.regionManager.invalidateStore(storeId);
recv.onStoreNotMatch(this.regionManager.getStoreById(storeId));
notifyStoreCacheInvalidate(storeId);
return true;
} else if (error.hasStaleEpoch()) {
// this error is reported from raftstore:
// region has outdated version，please try later.
logger.warn(String.format("Stale Epoch encountered for region [%s]", ctxRegion));
this.regionManager.onRegionStale(ctxRegion.getId());
notifyRegionCacheInvalidate(ctxRegion.getId());
return false;
} else if (error.hasServerIsBusy()) {
// this error is reported from kv:
// will occur when write pressure is high. Please try later.
logger.warn(
String.format(
"Server is busy for region [%s], reason: %s",
ctxRegion, error.getServerIsBusy().getReason()));
backOffer.doBackOff(
BackOffFunction.BackOffFuncType.BoServerBusy,
new StatusRuntimeException(
Status.fromCode(Status.Code.UNAVAILABLE).withDescription(error.toString())));
return true;
} else if (error.hasStaleCommand()) {
// this error is reported from raftstore:
// command outdated, please try later
logger.warn(String.format("Stale command for region [%s]", ctxRegion));
return true;
} else if (error.hasRaftEntryTooLarge()) {
logger.warn(String.format("Raft too large for region [%s]", ctxRegion));
throw new StatusRuntimeException(
Status.fromCode(Status.Code.UNAVAILABLE).withDescription(error.toString()));
} else if (error.hasKeyNotInRegion()) {
// this error is reported from raftstore:
// key requested is not in current region
// should not happen here.
ByteString invalidKey = error.getKeyNotInRegion().getKey();
logger.error(
String.format(
"Key not in region [%s] for key [%s], this error should not happen here.",
ctxRegion, KeyUtils.formatBytes(invalidKey)));
throw new StatusRuntimeException(Status.UNKNOWN.withDescription(error.toString()));
}
logger.warn(String.format("Unknown error for region [%s]", ctxRegion));
// For other errors, we only drop cache here.
// Upper level may split this task.
invalidateRegionStoreCache(ctxRegion);
}
return false;
}
@Override
public boolean handleRequestError(BackOffer backOffer, Exception e) {
regionManager.onRequestFail(ctxRegion.getId(), ctxRegion.getLeader().getStoreId());
notifyRegionStoreCacheInvalidate(
ctxRegion.getId(),
ctxRegion.getLeader().getStoreId(),
CacheInvalidateEvent.CacheType.REQ_FAILED);
backOffer.doBackOff(
BackOffFunction.BackOffFuncType.BoTiKVRPC,
new GrpcException(
"send tikv request error: " + e.getMessage() + ", try next peer later", e));
return true;
}
}

56
src/main/java/com/pingcap/tikv/operation/PDErrorHandler.java Normal file
View File

@ -0,0 +1,56 @@
/*
*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.pingcap.tikv.operation;
import com.pingcap.tikv.PDClient;
import com.pingcap.tikv.exception.GrpcException;
import com.pingcap.tikv.kvproto.Pdpb;
import com.pingcap.tikv.util.BackOffFunction;
import com.pingcap.tikv.util.BackOffer;
import java.util.function.Function;
public class PDErrorHandler<RespT> implements ErrorHandler<RespT> {
private final Function<RespT, Pdpb.Error> getError;
private final PDClient client;
public PDErrorHandler(Function<RespT, Pdpb.Error> errorExtractor, PDClient client) {
this.getError = errorExtractor;
this.client = client;
}
@Override
public boolean handleResponseError(BackOffer backOffer, RespT resp) {
if (resp == null) {
return false;
}
Pdpb.Error error = getError.apply(resp);
if (error != null) {
client.updateLeader();
backOffer.doBackOff(
BackOffFunction.BackOffFuncType.BoPDRPC, new GrpcException(error.toString()));
return true;
}
return false;
}
@Override
public boolean handleRequestError(BackOffer backOffer, Exception e) {
backOffer.doBackOff(BackOffFunction.BackOffFuncType.BoPDRPC, e);
return true;
}
}

123
src/main/java/com/pingcap/tikv/operation/SchemaInfer.java Normal file
View File

@ -0,0 +1,123 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.operation;
import com.pingcap.tikv.expression.ByItem;
import com.pingcap.tikv.expression.Expression;
import com.pingcap.tikv.meta.TiDAGRequest;
import com.pingcap.tikv.operation.transformer.Cast;
import com.pingcap.tikv.operation.transformer.NoOp;
import com.pingcap.tikv.operation.transformer.RowTransformer;
import com.pingcap.tikv.types.DataType;
import com.pingcap.tikv.types.IntegerType;
import com.pingcap.tikv.util.Pair;
import java.util.ArrayList;
import java.util.List;
/**
* SchemaInfer extract row's type after query is executed. It is pretty rough version. Optimization
* is on the way. The problem we have right now is that TiDB promote Sum to Decimal which is not
* compatible with column's type. The solution we come up with right now is use record column's type
* ad finalFieldType and build another list recording TiExpr's type as fieldType for row reading.
* Once we finish row reading, we first check each element in fieldType and finalFieldType share the
* same type or not. If yes, no need for casting. If no, casting is needed here.
*/
public class SchemaInfer {
private List<DataType> types;
private RowTransformer rt;
public static SchemaInfer create(TiDAGRequest dagRequest) {
return new SchemaInfer(dagRequest);
}
protected SchemaInfer(TiDAGRequest dagRequest) {
types = new ArrayList<>();
extractFieldTypes(dagRequest);
extractHandleType(dagRequest);
buildTransform(dagRequest);
}
private void extractHandleType(TiDAGRequest dagRequest) {
if (dagRequest.isHandleNeeded()) {
// DataType of handle is long
types.add(IntegerType.INT);
}
}
private void buildTransform(TiDAGRequest dagRequest) {
RowTransformer.Builder rowTrans = RowTransformer.newBuilder();
// Update:
// Switching to DAG mode will eliminate first blob
// TODO:check correctness of ↑
// 1. if group by is empty, first column should be "single group"
// which is a string
// 2. if multiple group by items present, it is wrapped inside
// a byte array. we make a multiple decoding
// 3. for no aggregation case, make only projected columns
// append aggregates if present
if (dagRequest.hasAggregate()) {
for (Pair<Expression, DataType> pair : dagRequest.getAggregatePairs()) {
rowTrans.addProjection(new Cast(pair.second));
}
if (dagRequest.hasGroupBy()) {
for (ByItem byItem : dagRequest.getGroupByItems()) {
rowTrans.addProjection(new NoOp(dagRequest.getExpressionType(byItem.getExpr())));
}
}
} else {
for (Expression field : dagRequest.getFields()) {
rowTrans.addProjection(new NoOp(dagRequest.getExpressionType(field)));
}
}
rowTrans.addSourceFieldTypes(types);
rt = rowTrans.build();
}
/**
* TODO: order by extract field types from tiSelectRequest for reading data to row.
*
* @param dagRequest is SelectRequest
*/
private void extractFieldTypes(TiDAGRequest dagRequest) {
if (dagRequest.hasAggregate()) {
dagRequest.getAggregates().forEach(expr -> types.add(dagRequest.getExpressionType(expr)));
// In DAG mode, if there is any group by statement in a request, all the columns specified
// in group by expression will be returned, so when we decode a result row, we need to pay
// extra attention to decoding.
if (dagRequest.hasGroupBy()) {
for (ByItem item : dagRequest.getGroupByItems()) {
types.add(dagRequest.getExpressionType(item.getExpr()));
}
}
} else {
// Extract all column type information from TiExpr
dagRequest.getFields().forEach(expr -> types.add(expr.getType()));
}
}
public DataType getType(int index) {
return types.get(index);
}
public List<DataType> getTypes() {
return types;
}
public RowTransformer getRowTransformer() {
return this.rt;
}
}

105
src/main/java/com/pingcap/tikv/operation/iterator/ChunkIterator.java Normal file
View File

@ -0,0 +1,105 @@
/*
*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.pingcap.tikv.operation.iterator;
import com.google.protobuf.ByteString;
import com.pingcap.tidb.tipb.Chunk;
import com.pingcap.tikv.exception.TiClientInternalException;
import java.util.Iterator;
import java.util.List;
public abstract class ChunkIterator<T> implements Iterator<T> {
private final List<Chunk> chunks;
protected int chunkIndex;
protected int metaIndex;
protected int bufOffset;
protected boolean eof;
public static ChunkIterator<ByteString> getRawBytesChunkIterator(List<Chunk> chunks) {
return new ChunkIterator<ByteString>(chunks) {
@Override
public ByteString next() {
Chunk c = chunks.get(chunkIndex);
long endOffset = c.getRowsMeta(metaIndex).getLength() + bufOffset;
if (endOffset > Integer.MAX_VALUE) {
throw new TiClientInternalException("Offset exceeded MAX_INT.");
}
ByteString result = c.getRowsData().substring(bufOffset, (int) endOffset);
advance();
return result;
}
};
}
public static ChunkIterator<Long> getHandleChunkIterator(List<Chunk> chunks) {
return new ChunkIterator<Long>(chunks) {
@Override
public Long next() {
Chunk c = chunks.get(chunkIndex);
long result = c.getRowsMeta(metaIndex).getHandle();
advance();
return result;
}
};
}
protected ChunkIterator(List<Chunk> chunks) {
// Read and then advance semantics
this.chunks = chunks;
this.chunkIndex = 0;
this.metaIndex = 0;
this.bufOffset = 0;
if (chunks.size() == 0
|| chunks.get(0).getRowsMetaCount() == 0
|| chunks.get(0).getRowsData().size() == 0) {
eof = true;
}
}
@Override
public boolean hasNext() {
return !eof;
}
private boolean seekNextNonEmptyChunk() {
// loop until the end of chunk list or first non empty chunk
do {
chunkIndex += 1;
} while (chunkIndex < chunks.size() && chunks.get(chunkIndex).getRowsMetaCount() == 0);
// return if remaining things left
return chunkIndex < chunks.size();
}
protected void advance() {
if (eof) {
return;
}
Chunk c = chunks.get(chunkIndex);
bufOffset += c.getRowsMeta(metaIndex++).getLength();
if (metaIndex >= c.getRowsMetaCount()) {
if (seekNextNonEmptyChunk()) {
metaIndex = 0;
bufOffset = 0;
} else {
eof = true;
}
}
}
}

46
src/main/java/com/pingcap/tikv/operation/iterator/ConcreteScanIterator.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright 2018 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.operation.iterator;
import com.google.protobuf.ByteString;
import com.pingcap.tikv.TiSession;
import com.pingcap.tikv.kvproto.Metapb;
import com.pingcap.tikv.region.RegionStoreClient;
import com.pingcap.tikv.region.TiRegion;
import com.pingcap.tikv.util.BackOffer;
import com.pingcap.tikv.util.ConcreteBackOffer;
import com.pingcap.tikv.util.Pair;
public class ConcreteScanIterator extends ScanIterator {
private final long version;
public ConcreteScanIterator(ByteString startKey, TiSession session, long version) {
// Passing endKey as ByteString.EMPTY means that endKey is +INF by default,
super(startKey, ByteString.EMPTY, Integer.MAX_VALUE, session);
this.version = version;
}
TiRegion loadCurrentRegionToCache() throws Exception {
Pair<TiRegion, Metapb.Store> pair = regionCache.getRegionStorePairByKey(startKey);
TiRegion region = pair.first;
Metapb.Store store = pair.second;
try (RegionStoreClient client = RegionStoreClient.create(region, store, session)) {
BackOffer backOffer = ConcreteBackOffer.newScannerNextMaxBackOff();
currentCache = client.scan(backOffer, startKey, version);
return region;
}
}
}

111
src/main/java/com/pingcap/tikv/operation/iterator/CoprocessIterator.java Normal file
View File

@ -0,0 +1,111 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.operation.iterator;
import static java.util.Objects.requireNonNull;
import com.pingcap.tidb.tipb.Chunk;
import com.pingcap.tidb.tipb.DAGRequest;
import com.pingcap.tikv.TiSession;
import com.pingcap.tikv.codec.CodecDataInput;
import com.pingcap.tikv.meta.TiDAGRequest;
import com.pingcap.tikv.operation.SchemaInfer;
import com.pingcap.tikv.row.Row;
import com.pingcap.tikv.row.RowReader;
import com.pingcap.tikv.row.RowReaderFactory;
import com.pingcap.tikv.types.DataType;
import com.pingcap.tikv.types.IntegerType;
import com.pingcap.tikv.util.RangeSplitter.RegionTask;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
public abstract class CoprocessIterator<T> implements Iterator<T> {
protected final TiSession session;
protected final List<RegionTask> regionTasks;
protected DAGRequest dagRequest;
protected static final DataType[] handleTypes = new DataType[] {IntegerType.INT};
// protected final ExecutorCompletionService<Iterator<SelectResponse>> completionService;
protected RowReader rowReader;
protected CodecDataInput dataInput;
protected boolean eof = false;
protected int taskIndex;
protected int chunkIndex;
protected List<Chunk> chunkList;
protected SchemaInfer schemaInfer;
CoprocessIterator(
DAGRequest req, List<RegionTask> regionTasks, TiSession session, SchemaInfer infer) {
this.dagRequest = req;
this.session = session;
this.regionTasks = regionTasks;
this.schemaInfer = infer;
}
abstract void submitTasks();
public static CoprocessIterator<Row> getRowIterator(
TiDAGRequest req, List<RegionTask> regionTasks, TiSession session) {
return new DAGIterator<Row>(
req.buildScan(req.isIndexScan() && !req.isDoubleRead()),
regionTasks,
session,
SchemaInfer.create(req),
req.getPushDownType()) {
@Override
public Row next() {
if (hasNext()) {
return rowReader.readRow(schemaInfer.getTypes().toArray(new DataType[0]));
} else {
throw new NoSuchElementException();
}
}
};
}
public static CoprocessIterator<Long> getHandleIterator(
TiDAGRequest req, List<RegionTask> regionTasks, TiSession session) {
return new DAGIterator<Long>(
req.buildScan(true), regionTasks, session, SchemaInfer.create(req), req.getPushDownType()) {
@Override
public Long next() {
if (hasNext()) {
return rowReader.readRow(handleTypes).getLong(0);
} else {
throw new NoSuchElementException();
}
}
};
}
boolean tryAdvanceChunkIndex() {
if (chunkList == null || chunkIndex >= chunkList.size() - 1) {
return false;
}
chunkIndex++;
return true;
}
void createDataInputReader() {
requireNonNull(chunkList, "Chunk list should not be null.");
if (0 > chunkIndex || chunkIndex >= chunkList.size()) {
throw new IllegalArgumentException();
}
dataInput = new CodecDataInput(chunkList.get(chunkIndex).getRowsData());
rowReader = RowReaderFactory.createRowReader(dataInput);
}
}

220
src/main/java/com/pingcap/tikv/operation/iterator/DAGIterator.java Normal file
View File

@ -0,0 +1,220 @@
package com.pingcap.tikv.operation.iterator;
import static com.pingcap.tikv.meta.TiDAGRequest.PushDownType.STREAMING;
import com.pingcap.tidb.tipb.Chunk;
import com.pingcap.tidb.tipb.DAGRequest;
import com.pingcap.tidb.tipb.SelectResponse;
import com.pingcap.tikv.TiSession;
import com.pingcap.tikv.exception.RegionTaskException;
import com.pingcap.tikv.exception.TiClientInternalException;
import com.pingcap.tikv.kvproto.Coprocessor;
import com.pingcap.tikv.kvproto.Metapb;
import com.pingcap.tikv.meta.TiDAGRequest.PushDownType;
import com.pingcap.tikv.operation.SchemaInfer;
import com.pingcap.tikv.region.RegionStoreClient;
import com.pingcap.tikv.region.TiRegion;
import com.pingcap.tikv.util.BackOffer;
import com.pingcap.tikv.util.ConcreteBackOffer;
import com.pingcap.tikv.util.RangeSplitter;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ExecutorCompletionService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public abstract class DAGIterator<T> extends CoprocessIterator<T> {
private ExecutorCompletionService<Iterator<SelectResponse>> streamingService;
private ExecutorCompletionService<SelectResponse> dagService;
private SelectResponse response;
private static final Logger logger = LoggerFactory.getLogger(DAGIterator.class.getName());
private Iterator<SelectResponse> responseIterator;
private final PushDownType pushDownType;
DAGIterator(
DAGRequest req,
List<RangeSplitter.RegionTask> regionTasks,
TiSession session,
SchemaInfer infer,
PushDownType pushDownType) {
super(req, regionTasks, session, infer);
this.pushDownType = pushDownType;
switch (pushDownType) {
case NORMAL:
dagService = new ExecutorCompletionService<>(session.getThreadPoolForTableScan());
break;
case STREAMING:
streamingService = new ExecutorCompletionService<>(session.getThreadPoolForTableScan());
break;
}
submitTasks();
}
@Override
void submitTasks() {
for (RangeSplitter.RegionTask task : regionTasks) {
switch (pushDownType) {
case STREAMING:
streamingService.submit(() -> processByStreaming(task));
break;
case NORMAL:
dagService.submit(() -> process(task));
break;
}
}
}
@Override
public boolean hasNext() {
if (eof) {
return false;
}
while (chunkList == null || chunkIndex >= chunkList.size() || dataInput.available() <= 0) {
// First we check if our chunk list has remaining chunk
if (tryAdvanceChunkIndex()) {
createDataInputReader();
}
// If not, check next region/response
else if (pushDownType == STREAMING) {
if (!advanceNextResponse() && !readNextRegionChunks()) {
return false;
}
} else if (!readNextRegionChunks()) {
return false;
}
}
return true;
}
private boolean hasMoreResponse() {
switch (pushDownType) {
case STREAMING:
return responseIterator != null && responseIterator.hasNext();
case NORMAL:
return response != null;
}
throw new IllegalArgumentException("Invalid push down type:" + pushDownType);
}
private boolean advanceNextResponse() {
if (!hasMoreResponse()) {
return false;
}
switch (pushDownType) {
case STREAMING:
chunkList = responseIterator.next().getChunksList();
break;
case NORMAL:
chunkList = response.getChunksList();
break;
}
if (chunkList == null || chunkList.isEmpty()) {
return false;
}
chunkIndex = 0;
createDataInputReader();
return true;
}
private boolean readNextRegionChunks() {
if (eof || regionTasks == null || taskIndex >= regionTasks.size()) {
return false;
}
try {
switch (pushDownType) {
case STREAMING:
responseIterator = streamingService.take().get();
break;
case NORMAL:
response = dagService.take().get();
break;
}
} catch (Exception e) {
throw new TiClientInternalException("Error reading region:", e);
}
taskIndex++;
return advanceNextResponse();
}
private SelectResponse process(RangeSplitter.RegionTask regionTask) {
Queue<RangeSplitter.RegionTask> remainTasks = new ArrayDeque<>();
Queue<SelectResponse> responseQueue = new ArrayDeque<>();
remainTasks.add(regionTask);
BackOffer backOffer = ConcreteBackOffer.newCopNextMaxBackOff();
// In case of one region task spilt into several others, we ues a queue to properly handle all
// the remaining tasks.
while (!remainTasks.isEmpty()) {
RangeSplitter.RegionTask task = remainTasks.poll();
if (task == null) continue;
List<Coprocessor.KeyRange> ranges = task.getRanges();
TiRegion region = task.getRegion();
Metapb.Store store = task.getStore();
try {
RegionStoreClient client = RegionStoreClient.create(region, store, session);
Collection<RangeSplitter.RegionTask> tasks =
client.coprocess(backOffer, dagRequest, ranges, responseQueue);
if (tasks != null) {
remainTasks.addAll(tasks);
}
} catch (Throwable e) {
// Handle region task failed
logger.error(
"Process region tasks failed, remain "
+ remainTasks.size()
+ " tasks not executed due to",
e);
// Rethrow to upper levels
eof = true;
throw new RegionTaskException("Handle region task failed:", e);
}
}
// Add all chunks to the final result
List<Chunk> resultChunk = new ArrayList<>();
while (!responseQueue.isEmpty()) {
SelectResponse response = responseQueue.poll();
if (response != null) {
resultChunk.addAll(response.getChunksList());
}
}
return SelectResponse.newBuilder().addAllChunks(resultChunk).build();
}
private Iterator<SelectResponse> processByStreaming(RangeSplitter.RegionTask regionTask) {
List<Coprocessor.KeyRange> ranges = regionTask.getRanges();
TiRegion region = regionTask.getRegion();
Metapb.Store store = regionTask.getStore();
RegionStoreClient client;
try {
client = RegionStoreClient.create(region, store, session);
Iterator<SelectResponse> responseIterator = client.coprocessStreaming(dagRequest, ranges);
if (responseIterator == null) {
eof = true;
return null;
}
return responseIterator;
} catch (Exception e) {
// TODO: Fix stale error handling in streaming
// see:https://github.com/pingcap/tikv-client-lib-java/pull/149
throw new TiClientInternalException("Error Closing Store client.", e);
}
}
}

105
src/main/java/com/pingcap/tikv/operation/iterator/IndexScanIterator.java Normal file
View File

@ -0,0 +1,105 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.operation.iterator;
import com.pingcap.tikv.Snapshot;
import com.pingcap.tikv.TiConfiguration;
import com.pingcap.tikv.TiSession;
import com.pingcap.tikv.exception.TiClientInternalException;
import com.pingcap.tikv.meta.TiDAGRequest;
import com.pingcap.tikv.row.Row;
import com.pingcap.tikv.util.RangeSplitter;
import com.pingcap.tikv.util.RangeSplitter.RegionTask;
import gnu.trove.list.array.TLongArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.concurrent.ExecutorCompletionService;
public class IndexScanIterator implements Iterator<Row> {
private final Iterator<Long> handleIterator;
private final TiDAGRequest dagReq;
private final Snapshot snapshot;
private Iterator<Row> rowIterator;
private final ExecutorCompletionService<Iterator<Row>> completionService;
private int batchCount = 0;
private final int batchSize;
public IndexScanIterator(Snapshot snapshot, TiDAGRequest req, Iterator<Long> handleIterator) {
TiSession session = snapshot.getSession();
TiConfiguration conf = session.getConf();
this.dagReq = req;
this.handleIterator = handleIterator;
this.snapshot = snapshot;
this.batchSize = conf.getIndexScanBatchSize();
this.completionService = new ExecutorCompletionService<>(session.getThreadPoolForIndexScan());
}
private TLongArrayList feedBatch() {
TLongArrayList handles = new TLongArrayList(512);
while (handleIterator.hasNext()) {
handles.add(handleIterator.next());
if (batchSize <= handles.size()) {
break;
}
}
return handles;
}
@Override
public boolean hasNext() {
try {
if (rowIterator == null) {
TiSession session = snapshot.getSession();
while (handleIterator.hasNext()) {
TLongArrayList handles = feedBatch();
batchCount++;
completionService.submit(
() -> {
List<RegionTask> tasks =
RangeSplitter.newSplitter(session.getRegionManager())
.splitAndSortHandlesByRegion(dagReq.getTableInfo().getId(), handles);
return CoprocessIterator.getRowIterator(dagReq, tasks, session);
});
}
while (batchCount > 0) {
rowIterator = completionService.take().get();
batchCount--;
if (rowIterator.hasNext()) {
return true;
}
}
}
if (rowIterator == null) {
return false;
}
} catch (Exception e) {
throw new TiClientInternalException("Error reading rows from handle", e);
}
return rowIterator.hasNext();
}
@Override
public Row next() {
if (hasNext()) {
return rowIterator.next();
} else {
throw new NoSuchElementException();
}
}
}

64
src/main/java/com/pingcap/tikv/operation/iterator/RawScanIterator.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.operation.iterator;
import com.google.protobuf.ByteString;
import com.pingcap.tikv.TiSession;
import com.pingcap.tikv.key.Key;
import com.pingcap.tikv.kvproto.Metapb;
import com.pingcap.tikv.region.RegionStoreClient;
import com.pingcap.tikv.region.TiRegion;
import com.pingcap.tikv.util.BackOffer;
import com.pingcap.tikv.util.ConcreteBackOffer;
import com.pingcap.tikv.util.Pair;
public class RawScanIterator extends ScanIterator {
public RawScanIterator(ByteString startKey, ByteString endKey, int limit, TiSession session) {
super(startKey, endKey, limit, session);
}
TiRegion loadCurrentRegionToCache() throws Exception {
Pair<TiRegion, Metapb.Store> pair = regionCache.getRegionStorePairByRawKey(startKey);
TiRegion region = pair.first;
Metapb.Store store = pair.second;
try (RegionStoreClient client = RegionStoreClient.create(region, store, session)) {
BackOffer backOffer = ConcreteBackOffer.newScannerNextMaxBackOff();
if (limit <= 0) {
currentCache = null;
} else {
currentCache = client.rawScan(backOffer, startKey, limit);
}
return region;
}
}
private boolean notEndOfScan() {
return limit > 0
&& !(lastBatch
&& (index >= currentCache.size()
|| Key.toRawKey(currentCache.get(index).getKey()).compareTo(endKey) >= 0));
}
@Override
public boolean hasNext() {
if (isCacheDrained() && cacheLoadFails()) {
endOfScan = true;
return false;
}
return notEndOfScan();
}
}

137
src/main/java/com/pingcap/tikv/operation/iterator/ScanIterator.java Normal file
View File

@ -0,0 +1,137 @@
/*
* Copyright 2018 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.operation.iterator;
import static java.util.Objects.requireNonNull;
import com.google.protobuf.ByteString;
import com.pingcap.tikv.TiSession;
import com.pingcap.tikv.exception.TiClientInternalException;
import com.pingcap.tikv.key.Key;
import com.pingcap.tikv.kvproto.Kvrpcpb;
import com.pingcap.tikv.region.RegionManager;
import com.pingcap.tikv.region.TiRegion;
import java.util.Iterator;
import java.util.List;
public abstract class ScanIterator implements Iterator<Kvrpcpb.KvPair> {
protected final TiSession session;
protected final RegionManager regionCache;
protected List<Kvrpcpb.KvPair> currentCache;
protected ByteString startKey;
protected int index = -1;
protected int limit;
protected boolean endOfScan = false;
protected Key endKey;
protected boolean hasEndKey;
protected boolean lastBatch = false;
ScanIterator(ByteString startKey, ByteString endKey, int limit, TiSession session) {
this.startKey = requireNonNull(startKey, "start key is null");
if (startKey.isEmpty()) {
throw new IllegalArgumentException("start key cannot be empty");
}
this.endKey = Key.toRawKey(requireNonNull(endKey, "end key is null"));
this.hasEndKey = !endKey.equals(ByteString.EMPTY);
this.limit = limit;
this.session = session;
this.regionCache = session.getRegionManager();
}
abstract TiRegion loadCurrentRegionToCache() throws Exception;
// return true if current cache is not loaded or empty
boolean cacheLoadFails() {
if (endOfScan || lastBatch) {
return true;
}
if (startKey.isEmpty()) {
return true;
}
try {
TiRegion region = loadCurrentRegionToCache();
ByteString curRegionEndKey = region.getEndKey();
// currentCache is null means no keys found, whereas currentCache is empty means no values
// found
// the difference lies in whether to continue scanning, because chances are that the same key
// is
// split in another region because of pending entries, region split, e.t.c.
// See https://github.com/pingcap/tispark/issues/393 for details
if (currentCache == null) {
return true;
}
index = 0;
Key lastKey = Key.EMPTY;
// Session should be single-threaded itself
// so that we don't worry about conf change in the middle
// of a transaction. Otherwise below code might lose data
if (currentCache.size() < session.getConf().getScanBatchSize()) {
startKey = curRegionEndKey;
} else {
// Start new scan from exact next key in current region
lastKey = Key.toRawKey(currentCache.get(currentCache.size() - 1).getKey());
startKey = lastKey.next().toByteString();
}
// notify last batch if lastKey is greater than or equal to endKey
if (hasEndKey && lastKey.compareTo(endKey) >= 0) {
lastBatch = true;
startKey = null;
}
} catch (Exception e) {
throw new TiClientInternalException("Error scanning data from region.", e);
}
return false;
}
boolean isCacheDrained() {
return currentCache == null || limit <= 0 || index >= currentCache.size() || index == -1;
}
@Override
public boolean hasNext() {
if (isCacheDrained() && cacheLoadFails()) {
endOfScan = true;
return false;
}
return true;
}
private Kvrpcpb.KvPair getCurrent() {
if (isCacheDrained()) {
return null;
}
if (index < currentCache.size()) {
--limit;
return currentCache.get(index++);
}
return null;
}
@Override
public Kvrpcpb.KvPair next() {
Kvrpcpb.KvPair kv = getCurrent();
if (kv == null) {
// cache drained
if (cacheLoadFails()) {
return null;
}
return getCurrent();
}
return kv;
}
}

97
src/main/java/com/pingcap/tikv/operation/transformer/Cast.java Normal file
View File

@ -0,0 +1,97 @@
/*
*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.pingcap.tikv.operation.transformer;
import com.pingcap.tikv.row.Row;
import com.pingcap.tikv.types.*;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
public class Cast extends NoOp {
public Cast(DataType type) {
super(type);
}
@Override
public void set(Object value, Row row, int pos) {
Object casted;
if (value == null) {
row.set(pos, targetDataType, null);
return;
}
if (targetDataType instanceof IntegerType) {
casted = castToLong(value);
} else if (targetDataType instanceof StringType) {
casted = castToString(value);
} else if (targetDataType instanceof BytesType) {
casted = castToBinary(value);
} else if (targetDataType instanceof DecimalType) {
casted = castToDecimal(value);
} else if (targetDataType instanceof RealType) {
casted = castToDouble(value);
} else {
casted = value;
}
row.set(pos, targetDataType, casted);
}
private Double castToDouble(Object obj) {
if (obj instanceof Number) {
Number num = (Number) obj;
return num.doubleValue();
}
throw new UnsupportedOperationException("can not cast un-number to double ");
}
private BigDecimal castToDecimal(Object obj) {
if (obj instanceof Number) {
Number num = (Number) obj;
return new BigDecimal(num.doubleValue());
}
throw new UnsupportedOperationException(
"Cannot cast to BigDecimal: " + (obj == null ? "null" : obj.getClass().getSimpleName()));
}
private Long castToLong(Object obj) {
if (obj instanceof Number) {
Number num = (Number) obj;
return num.longValue();
}
throw new UnsupportedOperationException("can not cast un-number to long ");
}
private String castToString(Object obj) {
String result;
if (obj instanceof byte[]) {
result = new String((byte[]) obj, StandardCharsets.UTF_8);
} else if (obj instanceof char[]) {
result = new String((char[]) obj);
} else {
result = String.valueOf(obj);
}
return result;
}
private byte[] castToBinary(Object obj) {
if (obj instanceof byte[]) {
return (byte[]) obj;
} else {
return obj.toString().getBytes();
}
}
}

60
src/main/java/com/pingcap/tikv/operation/transformer/MultiKeyDecoder.java Normal file
View File

@ -0,0 +1,60 @@
/*
*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.pingcap.tikv.operation.transformer;
import static java.util.Objects.requireNonNull;
import com.google.common.collect.ImmutableList;
import com.pingcap.tikv.codec.CodecDataInput;
import com.pingcap.tikv.row.Row;
import com.pingcap.tikv.types.DataType;
import java.util.List;
public class MultiKeyDecoder implements Projection {
public MultiKeyDecoder(List<DataType> dataTypes) {
this.resultTypes = requireNonNull(dataTypes).toArray(new DataType[0]);
}
private DataType[] resultTypes;
@Override
public void set(Object value, Row row, int pos) {
byte[] rowData = (byte[]) value;
CodecDataInput cdi = new CodecDataInput(rowData);
for (int i = 0; i < resultTypes.length; i++) {
DataType type = resultTypes[i];
if (type.isNextNull(cdi)) {
cdi.readUnsignedByte();
row.setNull(i + pos);
} else {
row.set(i + pos, type, type.decode(cdi));
}
}
}
@Override
public int size() {
return resultTypes.length;
}
@Override
public List<DataType> getTypes() {
return ImmutableList.copyOf(resultTypes);
}
}

47
src/main/java/com/pingcap/tikv/operation/transformer/NoOp.java Normal file
View File

@ -0,0 +1,47 @@
/*
*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.pingcap.tikv.operation.transformer;
import com.google.common.collect.ImmutableList;
import com.pingcap.tikv.row.Row;
import com.pingcap.tikv.types.DataType;
import java.util.List;
/** Noop is a base type projection, it basically do nothing but copy. */
public class NoOp implements Projection {
protected DataType targetDataType;
public NoOp(DataType dataType) {
this.targetDataType = dataType;
}
@Override
public void set(Object value, Row row, int pos) {
row.set(pos, targetDataType, value);
}
@Override
public int size() {
return 1;
}
@Override
public List<DataType> getTypes() {
return ImmutableList.of(targetDataType);
}
}

30
src/main/java/com/pingcap/tikv/operation/transformer/Projection.java Normal file
View File

@ -0,0 +1,30 @@
/*
*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.pingcap.tikv.operation.transformer;
import com.pingcap.tikv.row.Row;
import com.pingcap.tikv.types.DataType;
import java.util.List;
public interface Projection {
void set(Object value, Row row, int pos);
int size();
List<DataType> getTypes();
}

131
src/main/java/com/pingcap/tikv/operation/transformer/RowTransformer.java Normal file
View File

@ -0,0 +1,131 @@
/*
*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.pingcap.tikv.operation.transformer;
import static java.util.Objects.requireNonNull;
import com.google.common.collect.ImmutableList;
import com.pingcap.tikv.row.ObjectRowImpl;
import com.pingcap.tikv.row.Row;
import com.pingcap.tikv.types.DataType;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
/**
* RowTransformer is used along with SchemaInfer and row and provide some operation. If you do not
* know your target FieldType, then you do not need use this interface. The reason we provide this
* interface is that sometime we need use it. Suppose we have a table t1 and have two column c1 and
* s1 select sum(c1) from t1 will return SingleGroup literally and sum(c1). SingleGroup should be
* skipped. Hence, skip operation is needed here. Another usage is that sum(c1)'s type is decimal no
* matter what real column type is. We need cast it to target type which is column's type. Hence,
* cast operation is needed. RowTransformer is executed after row is already read from
* CodecDataInput.
*/
public class RowTransformer {
public static Builder newBuilder() {
return new Builder();
}
/** A Builder can build a RowTransformer. */
public static class Builder {
private final List<Projection> projections = new ArrayList<>();
private final List<DataType> sourceTypes = new ArrayList<>();
public RowTransformer build() {
return new RowTransformer(sourceTypes, projections);
}
public Builder addProjection(Projection projection) {
this.projections.add(projection);
return this;
}
public Builder addProjections(Projection... projections) {
this.projections.addAll(Arrays.asList(projections));
return this;
}
public Builder addSourceFieldType(DataType fieldType) {
this.sourceTypes.add(fieldType);
return this;
}
public Builder addSourceFieldTypes(DataType... fieldTypes) {
this.sourceTypes.addAll(Arrays.asList(fieldTypes));
return this;
}
public Builder addSourceFieldTypes(List<DataType> fieldTypes) {
this.sourceTypes.addAll(fieldTypes);
return this;
}
}
private final List<Projection> projections;
private final List<DataType> sourceFieldTypes;
private RowTransformer(List<DataType> sourceTypes, List<Projection> projections) {
this.sourceFieldTypes = ImmutableList.copyOf(requireNonNull(sourceTypes));
this.projections = ImmutableList.copyOf(requireNonNull(projections));
}
/**
* Transforms input row to a output row according projections operator passed on creation of this
* RowTransformer.
*
* @param inRow input row that need to be transformed.
* @return a row that is already transformed.
*/
public Row transform(Row inRow) {
// After transform the length of row is probably not same as the input row.
// we need calculate the new length.
Row outRow = ObjectRowImpl.create(newRowLength());
int offset = 0;
for (int i = 0; i < inRow.fieldCount(); i++) {
Object inVal = inRow.get(i, sourceFieldTypes.get(i));
Projection p = getProjection(i);
p.set(inVal, outRow, offset);
offset += p.size();
}
return outRow;
}
private Projection getProjection(int index) {
return projections.get(index);
}
/**
* Collect output row's length.
*
* @return a int which is the new length of output row.
*/
private int newRowLength() {
return this.projections.stream().reduce(0, (sum, p) -> sum += p.size(), (s1, s2) -> s1 + s2);
}
public List<DataType> getTypes() {
return projections
.stream()
.flatMap(proj -> proj.getTypes().stream())
.collect(Collectors.toList());
}
}

40
src/main/java/com/pingcap/tikv/operation/transformer/Skip.java Normal file
View File

@ -0,0 +1,40 @@
/*
*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.pingcap.tikv.operation.transformer;
import com.google.common.collect.ImmutableList;
import com.pingcap.tikv.row.Row;
import com.pingcap.tikv.types.DataType;
import java.util.List;
public class Skip implements Projection {
public static final Skip SKIP_OP = new Skip();
@Override
public void set(Object value, Row row, int pos) {}
@Override
public int size() {
return 0;
}
@Override
public List<DataType> getTypes() {
return ImmutableList.of();
}
}

39
src/main/java/com/pingcap/tikv/policy/RetryMaxMs.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.policy;
import com.pingcap.tikv.operation.ErrorHandler;
import com.pingcap.tikv.util.BackOffer;
public class RetryMaxMs<T> extends RetryPolicy<T> {
private RetryMaxMs(ErrorHandler<T> handler, BackOffer backOffer) {
super(handler);
this.backOffer = backOffer;
}
public static class Builder<T> implements RetryPolicy.Builder<T> {
private BackOffer backOffer;
public Builder(BackOffer backOffer) {
this.backOffer = backOffer;
}
@Override
public RetryPolicy<T> create(ErrorHandler<T> handler) {
return new RetryMaxMs<>(handler, backOffer);
}
}
}

78
src/main/java/com/pingcap/tikv/policy/RetryPolicy.java Normal file
View File

@ -0,0 +1,78 @@
/*
* Copyright 2017 PingCAP, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.pingcap.tikv.policy;
import com.google.common.collect.ImmutableSet;
import com.pingcap.tikv.exception.GrpcException;
import com.pingcap.tikv.operation.ErrorHandler;
import com.pingcap.tikv.util.BackOffer;
import com.pingcap.tikv.util.ConcreteBackOffer;
import io.grpc.Status;
import java.util.concurrent.Callable;
public abstract class RetryPolicy<RespT> {
BackOffer backOffer = ConcreteBackOffer.newCopNextMaxBackOff();
// handles PD and TiKV's error.
private ErrorHandler<RespT> handler;
private ImmutableSet<Status.Code> unrecoverableStatus =
ImmutableSet.of(
Status.Code.ALREADY_EXISTS, Status.Code.PERMISSION_DENIED,
Status.Code.INVALID_ARGUMENT, Status.Code.NOT_FOUND,
Status.Code.UNIMPLEMENTED, Status.Code.OUT_OF_RANGE,
Status.Code.UNAUTHENTICATED, Status.Code.CANCELLED);
RetryPolicy(ErrorHandler<RespT> handler) {
this.handler = handler;
}
private void rethrowNotRecoverableException(Exception e) {
Status status = Status.fromThrowable(e);
if (unrecoverableStatus.contains(status.getCode())) {
throw new GrpcException(e);
}
}
public RespT callWithRetry(Callable<RespT> proc, String methodName) {
while (true) {
RespT result = null;
try {
result = proc.call();
} catch (Exception e) {
rethrowNotRecoverableException(e);
// Handle request call error
boolean retry = handler.handleRequestError(backOffer, e);
if (retry) {
continue;
}
}
// Handle response error
if (handler != null) {
boolean retry = handler.handleResponseError(backOffer, result);
if (retry) {
continue;
}
}
return result;
}
}
public interface Builder<T> {
RetryPolicy<T> create(ErrorHandler<T> handler);
}
}

Some files were not shown because too many files have changed in this diff Show More