diff --git a/pom.xml b/pom.xml index 3bb331df8a..ef2f274d59 100644 --- a/pom.xml +++ b/pom.xml @@ -34,93 +34,15 @@ - org.apache.logging.log4j - log4j-api - 2.8.1 - provided - - - org.apache.logging.log4j - log4j-core - 2.8.1 - provided - - - org.apache.spark - spark-core_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-catalyst_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-hive-thriftserver_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-unsafe_${scala.binary.version} - ${spark.version} - provided - - - org.slf4j - slf4j-api - ${slf4j.version} - provided - - - org.slf4j - slf4j-log4j12 - ${slf4j.version} - provided - - - org.slf4j - jul-to-slf4j - ${slf4j.version} - provided - - - org.slf4j - jcl-over-slf4j - ${slf4j.version} - provided + org.apache.commons + commons-lang3 + 3.5 log4j log4j ${log4j.version} - - - net.sf.trove4j - trove4j - ${trove4j.version} - - - com.sangupta - murmur - 1.0.0 - io.grpc @@ -137,37 +59,6 @@ grpc-stub ${grpc.version} - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.databind.version} - - - com.fasterxml.jackson.module - jackson-module-scala_${scala.binary.version} - ${jackson.version} - - - com.google.guava - guava - - - - - joda-time - joda-time - ${joda-time.version} - - - org.joda - joda-convert - ${joda-convert.version} - io.grpc grpc-testing @@ -220,7 +111,7 @@ ${proto.folder} - ${basedir}/kvproto/vendor/github.com/gogo/protobuf + ${basedir}/kvproto/include **/gogoproto/** diff --git a/src/main/java/org/tikv/PDClient.java b/src/main/java/org/tikv/PDClient.java index ad21c4c777..822aaa760e 100644 --- a/src/main/java/org/tikv/PDClient.java +++ b/src/main/java/org/tikv/PDClient.java @@ -34,7 +34,6 @@ import org.tikv.codec.Codec.BytesCodec; import org.tikv.codec.CodecDataOutput; import org.tikv.exception.GrpcException; import org.tikv.exception.TiClientInternalException; -import org.tikv.kvproto.Kvrpcpb.IsolationLevel; import org.tikv.kvproto.Metapb.Store; import org.tikv.kvproto.PDGrpc; import org.tikv.kvproto.PDGrpc.PDBlockingStub; @@ -62,7 +61,6 @@ public class PDClient extends AbstractGRPCClient private TsoRequest tsoReq; private volatile LeaderWrapper leaderWrapper; private ScheduledExecutorService service; - private IsolationLevel isolationLevel; private List pdAddrs; @Override diff --git a/src/main/java/org/tikv/Snapshot.java b/src/main/java/org/tikv/Snapshot.java index cd4d1dd9ed..cafe0a819d 100644 --- a/src/main/java/org/tikv/Snapshot.java +++ b/src/main/java/org/tikv/Snapshot.java @@ -15,8 +15,6 @@ package org.tikv; -import static org.tikv.operation.iterator.CoprocessIterator.getHandleIterator; -import static org.tikv.operation.iterator.CoprocessIterator.getRowIterator; import static org.tikv.util.KeyRangeUtils.makeRange; import com.google.common.collect.Range; @@ -28,18 +26,13 @@ import org.tikv.exception.TiClientInternalException; import org.tikv.key.Key; import org.tikv.kvproto.Kvrpcpb.KvPair; import org.tikv.kvproto.Metapb.Store; -import org.tikv.meta.TiDAGRequest; import org.tikv.meta.TiTimestamp; import org.tikv.operation.iterator.ConcreteScanIterator; -import org.tikv.operation.iterator.IndexScanIterator; import org.tikv.region.RegionStoreClient; import org.tikv.region.TiRegion; -import org.tikv.row.Row; import org.tikv.util.BackOffer; import org.tikv.util.ConcreteBackOffer; import org.tikv.util.Pair; -import org.tikv.util.RangeSplitter; -import org.tikv.util.RangeSplitter.RegionTask; public class Snapshot { private final TiTimestamp timestamp; @@ -77,59 +70,6 @@ public class Snapshot { return client.get(ConcreteBackOffer.newGetBackOff(), key, timestamp.getVersion()); } - /** - * Issue a table read request - * - * @param dagRequest DAG request for coprocessor - * @return a Iterator that contains all result from this select request. - */ - public Iterator tableRead(TiDAGRequest dagRequest) { - if (dagRequest.isIndexScan()) { - Iterator iter = - getHandleIterator( - dagRequest, - RangeSplitter.newSplitter(session.getRegionManager()) - .splitRangeByRegion(dagRequest.getRanges()), - session); - return new IndexScanIterator(this, dagRequest, iter); - } else { - return getRowIterator( - dagRequest, - RangeSplitter.newSplitter(session.getRegionManager()) - .splitRangeByRegion(dagRequest.getRanges()), - session); - } - } - - /** - * Below is lower level API for env like Spark which already did key range split Perform table - * scan - * - * @param dagRequest DAGRequest for coprocessor - * @param task RegionTask of the coprocessor request to send - * @return Row iterator to iterate over resulting rows - */ - public Iterator tableRead(TiDAGRequest dagRequest, List task) { - if (dagRequest.isDoubleRead()) { - Iterator iter = getHandleIterator(dagRequest, task, session); - return new IndexScanIterator(this, dagRequest, iter); - } else { - return getRowIterator(dagRequest, task, session); - } - } - - /** - * Below is lower level API for env like Spark which already did key range split Perform handle - * scan - * - * @param dagRequest DAGRequest for coprocessor - * @param tasks RegionTask of the coprocessor request to send - * @return Row iterator to iterate over resulting rows - */ - public Iterator indexHandleRead(TiDAGRequest dagRequest, List tasks) { - return getHandleIterator(dagRequest, tasks, session); - } - public Iterator scan(ByteString startKey) { return new ConcreteScanIterator(startKey, session, timestamp.getVersion()); } diff --git a/src/main/java/org/tikv/TiSession.java b/src/main/java/org/tikv/TiSession.java index 2c977c25ff..8110616f97 100644 --- a/src/main/java/org/tikv/TiSession.java +++ b/src/main/java/org/tikv/TiSession.java @@ -16,31 +16,19 @@ package org.tikv; import com.google.common.net.HostAndPort; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import org.tikv.catalog.Catalog; -import org.tikv.event.CacheInvalidateEvent; -import org.tikv.meta.TiTimestamp; import org.tikv.region.RegionManager; -import org.tikv.util.ConcreteBackOffer; public class TiSession implements AutoCloseable { private static final Map connPool = new HashMap<>(); private final TiConfiguration conf; - private Function cacheInvalidateCallback; // below object creation is either heavy or making connection (pd), pending for lazy loading private volatile RegionManager regionManager; private volatile PDClient client; - private volatile Catalog catalog; - private volatile ExecutorService indexScanThreadPool; - private volatile ExecutorService tableScanThreadPool; public TiSession(TiConfiguration conf) { this.conf = conf; @@ -50,18 +38,6 @@ public class TiSession implements AutoCloseable { return conf; } - public TiTimestamp getTimestamp() { - return getPDClient().getTimestamp(ConcreteBackOffer.newTsoBackOff()); - } - - public Snapshot createSnapshot() { - return new Snapshot(getTimestamp(), this); - } - - public Snapshot createSnapshot(TiTimestamp ts) { - return new Snapshot(ts, this); - } - public PDClient getPDClient() { PDClient res = client; if (res == null) { @@ -75,25 +51,6 @@ public class TiSession implements AutoCloseable { return res; } - public Catalog getCatalog() { - Catalog res = catalog; - if (res == null) { - synchronized (this) { - if (catalog == null) { - catalog = - new Catalog( - this::createSnapshot, - conf.getMetaReloadPeriod(), - conf.getMetaReloadPeriodUnit(), - conf.ifShowRowId(), - conf.getDBPrefix()); - } - res = catalog; - } - } - return res; - } - public synchronized RegionManager getRegionManager() { RegionManager res = regionManager; if (res == null) { @@ -130,58 +87,10 @@ public class TiSession implements AutoCloseable { return channel; } - public ExecutorService getThreadPoolForIndexScan() { - ExecutorService res = indexScanThreadPool; - if (res == null) { - synchronized (this) { - if (indexScanThreadPool == null) { - indexScanThreadPool = - Executors.newFixedThreadPool( - conf.getIndexScanConcurrency(), - new ThreadFactoryBuilder().setDaemon(true).build()); - } - res = indexScanThreadPool; - } - } - return res; - } - - public ExecutorService getThreadPoolForTableScan() { - ExecutorService res = tableScanThreadPool; - if (res == null) { - synchronized (this) { - if (tableScanThreadPool == null) { - tableScanThreadPool = - Executors.newFixedThreadPool( - conf.getTableScanConcurrency(), - new ThreadFactoryBuilder().setDaemon(true).build()); - } - res = tableScanThreadPool; - } - } - return res; - } - public static TiSession create(TiConfiguration conf) { return new TiSession(conf); } - public Function getCacheInvalidateCallback() { - return cacheInvalidateCallback; - } - - /** - * This is used for setting call back function to invalidate cache information - * - * @param callBackFunc callback function - */ - public void injectCallBackFunc(Function callBackFunc) { - this.cacheInvalidateCallback = callBackFunc; - } - @Override - public void close() throws Exception { - getThreadPoolForTableScan().shutdownNow(); - getThreadPoolForIndexScan().shutdownNow(); - } + public void close() throws Exception {} } diff --git a/src/main/java/org/tikv/catalog/Catalog.java b/src/main/java/org/tikv/catalog/Catalog.java deleted file mode 100644 index 80f3bf8198..0000000000 --- a/src/main/java/org/tikv/catalog/Catalog.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.catalog; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import org.apache.log4j.Logger; -import org.tikv.Snapshot; -import org.tikv.meta.TiDBInfo; -import org.tikv.meta.TiTableInfo; - -public class Catalog implements AutoCloseable { - private Supplier snapshotProvider; - private ScheduledExecutorService service; - private CatalogCache metaCache; - private final boolean showRowId; - private final String dbPrefix; - private final Logger logger = Logger.getLogger(this.getClass()); - - @Override - public void close() throws Exception { - if (service != null) { - service.shutdown(); - } - } - - private static class CatalogCache { - - private CatalogCache(CatalogTransaction transaction, String dbPrefix, boolean loadTables) { - this.transaction = transaction; - this.dbPrefix = dbPrefix; - this.dbCache = loadDatabases(loadTables); - this.tableCache = new ConcurrentHashMap<>(); - this.currentVersion = transaction.getLatestSchemaVersion(); - } - - private final Map dbCache; - private final ConcurrentHashMap> tableCache; - private CatalogTransaction transaction; - private long currentVersion; - private final String dbPrefix; - - public CatalogTransaction getTransaction() { - return transaction; - } - - public long getVersion() { - return currentVersion; - } - - public TiDBInfo getDatabase(String name) { - Objects.requireNonNull(name, "name is null"); - return dbCache.get(name.toLowerCase()); - } - - public List listDatabases() { - return ImmutableList.copyOf(dbCache.values()); - } - - public List listTables(TiDBInfo db) { - Map tableMap = tableCache.get(db); - if (tableMap == null) { - tableMap = loadTables(db); - } - return ImmutableList.copyOf(tableMap.values()); - } - - public TiTableInfo getTable(TiDBInfo db, String tableName) { - Map tableMap = tableCache.get(db); - if (tableMap == null) { - tableMap = loadTables(db); - } - return tableMap.get(tableName.toLowerCase()); - } - - private Map loadTables(TiDBInfo db) { - List tables = transaction.getTables(db.getId()); - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (TiTableInfo table : tables) { - builder.put(table.getName().toLowerCase(), table); - } - Map tableMap = builder.build(); - tableCache.put(db, tableMap); - return tableMap; - } - - private Map loadDatabases(boolean loadTables) { - HashMap newDBCache = new HashMap<>(); - - List databases = transaction.getDatabases(); - databases.forEach( - db -> { - TiDBInfo newDBInfo = db.rename(dbPrefix + db.getName()); - newDBCache.put(newDBInfo.getName().toLowerCase(), newDBInfo); - if (loadTables) { - loadTables(newDBInfo); - } - }); - return newDBCache; - } - } - - public Catalog( - Supplier snapshotProvider, - int refreshPeriod, - TimeUnit periodUnit, - boolean showRowId, - String dbPrefix) { - this.snapshotProvider = Objects.requireNonNull(snapshotProvider, "Snapshot Provider is null"); - this.showRowId = showRowId; - this.dbPrefix = dbPrefix; - metaCache = new CatalogCache(new CatalogTransaction(snapshotProvider.get()), dbPrefix, false); - service = - Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder().setDaemon(true).build()); - service.scheduleAtFixedRate( - () -> { - // Wrap this with a try catch block in case schedule update fails - try { - reloadCache(); - } catch (Exception e) { - logger.warn("Reload Cache failed", e); - } - }, - refreshPeriod, - refreshPeriod, - periodUnit); - } - - public void reloadCache(boolean loadTables) { - Snapshot snapshot = snapshotProvider.get(); - CatalogTransaction newTrx = new CatalogTransaction(snapshot); - long latestVersion = newTrx.getLatestSchemaVersion(); - if (latestVersion > metaCache.getVersion()) { - metaCache = new CatalogCache(newTrx, dbPrefix, loadTables); - } - } - - @VisibleForTesting - public void reloadCache() { - reloadCache(false); - } - - public List listDatabases() { - return metaCache.listDatabases(); - } - - public List listTables(TiDBInfo database) { - Objects.requireNonNull(database, "database is null"); - if (showRowId) { - return metaCache - .listTables(database) - .stream() - .map(TiTableInfo::copyTableWithRowId) - .collect(Collectors.toList()); - } else { - return metaCache.listTables(database); - } - } - - public TiDBInfo getDatabase(String dbName) { - Objects.requireNonNull(dbName, "dbName is null"); - return metaCache.getDatabase(dbName); - } - - public TiTableInfo getTable(String dbName, String tableName) { - TiDBInfo database = getDatabase(dbName); - if (database == null) { - return null; - } - return getTable(database, tableName); - } - - public TiTableInfo getTable(TiDBInfo database, String tableName) { - Objects.requireNonNull(database, "database is null"); - Objects.requireNonNull(tableName, "tableName is null"); - TiTableInfo table = metaCache.getTable(database, tableName); - if (showRowId) { - return table.copyTableWithRowId(); - } else { - return table; - } - } - - @VisibleForTesting - public TiTableInfo getTable(TiDBInfo database, long tableId) { - Objects.requireNonNull(database, "database is null"); - Collection tables = listTables(database); - for (TiTableInfo table : tables) { - if (table.getId() == tableId) { - if (showRowId) { - return table.copyTableWithRowId(); - } else { - return table; - } - } - } - return null; - } -} diff --git a/src/main/java/org/tikv/catalog/CatalogTransaction.java b/src/main/java/org/tikv/catalog/CatalogTransaction.java deleted file mode 100644 index 5dbaa9e88b..0000000000 --- a/src/main/java/org/tikv/catalog/CatalogTransaction.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.catalog; - -import static com.google.common.base.Preconditions.checkArgument; - -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; -import com.google.protobuf.ByteString; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import org.apache.log4j.Logger; -import org.tikv.Snapshot; -import org.tikv.codec.Codec.BytesCodec; -import org.tikv.codec.Codec.IntegerCodec; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.codec.KeyUtils; -import org.tikv.exception.TiClientInternalException; -import org.tikv.kvproto.Kvrpcpb; -import org.tikv.meta.TiDBInfo; -import org.tikv.meta.TiTableInfo; -import org.tikv.util.Pair; - -public class CatalogTransaction { - protected static final Logger logger = Logger.getLogger(Catalog.class); - private final Snapshot snapshot; - private final byte[] prefix; - - private static final byte[] META_PREFIX = new byte[] {'m'}; - - private static final byte HASH_DATA_FLAG = 'h'; - private static final byte STR_DATA_FLAG = 's'; - - private static ByteString KEY_DB = ByteString.copyFromUtf8("DBs"); - private static ByteString KEY_TABLE = ByteString.copyFromUtf8("Table"); - private static ByteString KEY_SCHEMA_VERSION = ByteString.copyFromUtf8("SchemaVersionKey"); - - private static final String ENCODED_DB_PREFIX = "DB"; - - public CatalogTransaction(Snapshot snapshot) { - this.snapshot = snapshot; - this.prefix = META_PREFIX; - } - - private void encodeStringDataKey(CodecDataOutput cdo, byte[] key) { - cdo.write(prefix); - BytesCodec.writeBytes(cdo, key); - IntegerCodec.writeULong(cdo, STR_DATA_FLAG); - } - - private void encodeHashDataKey(CodecDataOutput cdo, byte[] key, byte[] field) { - encodeHashDataKeyPrefix(cdo, key); - BytesCodec.writeBytes(cdo, field); - } - - private void encodeHashDataKeyPrefix(CodecDataOutput cdo, byte[] key) { - cdo.write(prefix); - BytesCodec.writeBytes(cdo, key); - IntegerCodec.writeULong(cdo, HASH_DATA_FLAG); - } - - private Pair decodeHashDataKey(ByteString rawKey) { - checkArgument( - KeyUtils.hasPrefix(rawKey, ByteString.copyFrom(prefix)), - "invalid encoded hash data key prefix: " + new String(prefix)); - CodecDataInput cdi = new CodecDataInput(rawKey.toByteArray()); - cdi.skipBytes(prefix.length); - byte[] key = BytesCodec.readBytes(cdi); - long typeFlag = IntegerCodec.readULong(cdi); - if (typeFlag != HASH_DATA_FLAG) { - throw new TiClientInternalException("Invalid hash data flag: " + typeFlag); - } - byte[] field = BytesCodec.readBytes(cdi); - return Pair.create(ByteString.copyFrom(key), ByteString.copyFrom(field)); - } - - private ByteString hashGet(ByteString key, ByteString field) { - CodecDataOutput cdo = new CodecDataOutput(); - encodeHashDataKey(cdo, key.toByteArray(), field.toByteArray()); - return snapshot.get(cdo.toByteString()); - } - - private ByteString bytesGet(ByteString key) { - CodecDataOutput cdo = new CodecDataOutput(); - encodeStringDataKey(cdo, key.toByteArray()); - return snapshot.get(cdo.toByteString()); - } - - private List> hashGetFields(ByteString key) { - CodecDataOutput cdo = new CodecDataOutput(); - encodeHashDataKeyPrefix(cdo, key.toByteArray()); - ByteString encodedKey = cdo.toByteString(); - - Iterator iterator = snapshot.scan(encodedKey); - List> fields = new ArrayList<>(); - while (iterator.hasNext()) { - Kvrpcpb.KvPair kv = iterator.next(); - if (!KeyUtils.hasPrefix(kv.getKey(), encodedKey)) { - break; - } - fields.add(Pair.create(decodeHashDataKey(kv.getKey()).second, kv.getValue())); - } - - return fields; - } - - private static ByteString encodeDatabaseID(long id) { - return ByteString.copyFrom(String.format("%s:%d", ENCODED_DB_PREFIX, id).getBytes()); - } - - public long getLatestSchemaVersion() { - ByteString versionBytes = bytesGet(KEY_SCHEMA_VERSION); - CodecDataInput cdi = new CodecDataInput(versionBytes.toByteArray()); - return Long.parseLong(new String(cdi.toByteArray(), StandardCharsets.UTF_8)); - } - - public List getDatabases() { - List> fields = hashGetFields(KEY_DB); - ImmutableList.Builder builder = ImmutableList.builder(); - for (Pair pair : fields) { - builder.add(parseFromJson(pair.second, TiDBInfo.class)); - } - return builder.build(); - } - - public TiDBInfo getDatabase(long id) { - ByteString dbKey = encodeDatabaseID(id); - ByteString json = hashGet(KEY_DB, dbKey); - if (json == null || json.isEmpty()) { - return null; - } - return parseFromJson(json, TiDBInfo.class); - } - - public List getTables(long dbId) { - ByteString dbKey = encodeDatabaseID(dbId); - List> fields = hashGetFields(dbKey); - ImmutableList.Builder builder = ImmutableList.builder(); - for (Pair pair : fields) { - if (KeyUtils.hasPrefix(pair.first, KEY_TABLE)) { - builder.add(parseFromJson(pair.second, TiTableInfo.class)); - } - } - return builder.build(); - } - - public static T parseFromJson(ByteString json, Class cls) { - Objects.requireNonNull(json, "json is null"); - Objects.requireNonNull(cls, "cls is null"); - - logger.debug(String.format("Parse Json %s : %s", cls.getSimpleName(), json.toStringUtf8())); - ObjectMapper mapper = new ObjectMapper(); - try { - return mapper.readValue(json.toStringUtf8(), cls); - } catch (JsonParseException | JsonMappingException e) { - String errMsg = - String.format( - "Invalid JSON value for Type %s: %s\n", cls.getSimpleName(), json.toStringUtf8()); - throw new TiClientInternalException(errMsg, e); - } catch (Exception e1) { - throw new TiClientInternalException("Error parsing Json", e1); - } - } -} diff --git a/src/main/java/org/tikv/codec/Codec.java b/src/main/java/org/tikv/codec/Codec.java index 8bd473b7e2..db9aa73b91 100644 --- a/src/main/java/org/tikv/codec/Codec.java +++ b/src/main/java/org/tikv/codec/Codec.java @@ -17,11 +17,7 @@ package org.tikv.codec; import static com.google.common.base.Preconditions.checkArgument; -import gnu.trove.list.array.TIntArrayList; -import java.math.BigDecimal; -import java.sql.Date; import java.util.Arrays; -import org.joda.time.*; import org.tikv.exception.InvalidCodecFormatException; public class Codec { @@ -319,338 +315,4 @@ public class Codec { return bytes; } } - - public static class RealCodec { - - private static final long signMask = 0x8000000000000000L; - - /** - * Decode as float - * - * @param cdi source of data - * @return decoded unsigned long value - */ - public static double readDouble(CodecDataInput cdi) { - long u = IntegerCodec.readULong(cdi); - if (u < 0) { - u &= Long.MAX_VALUE; - } else { - u = ~u; - } - return Double.longBitsToDouble(u); - } - - private static long encodeDoubleToCmpLong(double val) { - long u = Double.doubleToRawLongBits(val); - if (val >= 0) { - u |= signMask; - } else { - u = ~u; - } - return u; - } - - public static void writeDoubleFully(CodecDataOutput cdo, double val) { - cdo.writeByte(FLOATING_FLAG); - writeDouble(cdo, val); - } - - /** - * Encoding a double value to byte buffer - * - * @param cdo For outputting data in bytes array - * @param val The data to encode - */ - public static void writeDouble(CodecDataOutput cdo, double val) { - IntegerCodec.writeULong(cdo, encodeDoubleToCmpLong(val)); - } - } - - public static class DecimalCodec { - - /** - * read a decimal value from CodecDataInput - * - * @param cdi cdi is source data. - */ - public static BigDecimal readDecimal(CodecDataInput cdi) { - if (cdi.available() < 3) { - throw new IllegalArgumentException("insufficient bytes to read value"); - } - - // 64 should be larger enough for avoiding unnecessary growth. - TIntArrayList data = new TIntArrayList(64); - int precision = cdi.readUnsignedByte(); - int frac = cdi.readUnsignedByte(); - int length = precision + frac; - int curPos = cdi.size() - cdi.available(); - for (int i = 0; i < length; i++) { - if (cdi.eof()) { - break; - } - data.add(cdi.readUnsignedByte()); - } - - MyDecimal dec = new MyDecimal(); - int binSize = dec.fromBin(precision, frac, data.toArray()); - cdi.mark(curPos + binSize); - cdi.reset(); - return dec.toDecimal(); - } - - /** - * write a decimal value from CodecDataInput - * - * @param cdo cdo is destination data. - * @param dec is decimal value that will be written into cdo. - */ - static void writeDecimal(CodecDataOutput cdo, MyDecimal dec) { - int[] data = dec.toBin(dec.precision(), dec.frac()); - cdo.writeByte(dec.precision()); - cdo.writeByte(dec.frac()); - for (int aData : data) { - cdo.writeByte(aData & 0xFF); - } - } - - public static void writeDecimalFully(CodecDataOutput cdo, BigDecimal val) { - cdo.writeByte(DECIMAL_FLAG); - writeDecimal(cdo, val); - } - - /** - * Encoding a double value to byte buffer - * - * @param cdo For outputting data in bytes array - * @param val The data to encode - */ - public static void writeDecimal(CodecDataOutput cdo, BigDecimal val) { - MyDecimal dec = new MyDecimal(); - dec.fromString(val.toPlainString()); - writeDecimal(cdo, dec); - } - } - - public static class DateTimeCodec { - - /** - * Encode a DateTime to a packed long converting to specific timezone - * - * @param dateTime dateTime that need to be encoded. - * @param tz timezone used for converting to localDateTime - * @return a packed long. - */ - static long toPackedLong(DateTime dateTime, DateTimeZone tz) { - LocalDateTime localDateTime = dateTime.withZone(tz).toLocalDateTime(); - return toPackedLong( - localDateTime.getYear(), - localDateTime.getMonthOfYear(), - localDateTime.getDayOfMonth(), - localDateTime.getHourOfDay(), - localDateTime.getMinuteOfHour(), - localDateTime.getSecondOfMinute(), - localDateTime.getMillisOfSecond() * 1000); - } - - /** - * Encode a date/time parts to a packed long. - * - * @return a packed long. - */ - static long toPackedLong( - int year, int month, int day, int hour, int minute, int second, int micro) { - long ymd = (year * 13 + month) << 5 | day; - long hms = hour << 12 | minute << 6 | second; - return ((ymd << 17 | hms) << 24) | micro; - } - - /** - * Read datetime from packed Long which contains all parts of a datetime namely, year, month, - * day and hour, min and sec, millisec. The original representation does not indicate any - * timezone information In Timestamp type, it should be interpreted as UTC while in DateType it - * is interpreted as local timezone - * - * @param packed long value that packs date / time parts - * @param tz timezone to interpret datetime parts - * @return decoded DateTime using provided timezone - */ - static DateTime fromPackedLong(long packed, DateTimeZone tz) { - // TODO: As for JDBC behavior, it can be configured to "round" or "toNull" - // for now we didn't pass in session so we do a toNull behavior - if (packed == 0) { - return null; - } - long ymdhms = packed >> 24; - long ymd = ymdhms >> 17; - int day = (int) (ymd & ((1 << 5) - 1)); - long ym = ymd >> 5; - int month = (int) (ym % 13); - int year = (int) (ym / 13); - - int hms = (int) (ymdhms & ((1 << 17) - 1)); - int second = hms & ((1 << 6) - 1); - int minute = (hms >> 6) & ((1 << 6) - 1); - int hour = hms >> 12; - int microsec = (int) (packed % (1 << 24)); - - try { - return new DateTime(year, month, day, hour, minute, second, microsec / 1000, tz); - } catch (IllegalInstantException e) { - LocalDateTime localDateTime = - new LocalDateTime(year, month, day, hour, minute, second, microsec / 1000); - DateTime dt = localDateTime.toLocalDate().toDateTimeAtStartOfDay(tz); - long millis = dt.getMillis() + localDateTime.toLocalTime().getMillisOfDay(); - return new DateTime(millis, tz); - } - } - - /** - * Encode DateTime as packed long converting into specified timezone All timezone conversion - * should be done beforehand - * - * @param cdo encoding output - * @param dateTime value to encode - * @param tz timezone used to converting local time - */ - public static void writeDateTimeFully(CodecDataOutput cdo, DateTime dateTime, DateTimeZone tz) { - long val = DateTimeCodec.toPackedLong(dateTime, tz); - IntegerCodec.writeULongFully(cdo, val, true); - } - - /** - * Encode DateTime as packed long converting into specified timezone All timezone conversion - * should be done beforehand The encoded value has no data type flag - * - * @param cdo encoding output - * @param dateTime value to encode - * @param tz timezone used to converting local time - */ - public static void writeDateTimeProto(CodecDataOutput cdo, DateTime dateTime, DateTimeZone tz) { - long val = DateTimeCodec.toPackedLong(dateTime, tz); - IntegerCodec.writeULong(cdo, val); - } - - /** - * Read datetime from packed Long encoded as unsigned var-len integer converting into specified - * timezone - * - * @see DateTimeCodec#fromPackedLong(long, DateTimeZone) - * @param cdi codec buffer input - * @param tz timezone to interpret datetime parts - * @return decoded DateTime using provided timezone - */ - public static DateTime readFromUVarInt(CodecDataInput cdi, DateTimeZone tz) { - return DateTimeCodec.fromPackedLong(IntegerCodec.readUVarLong(cdi), tz); - } - - /** - * Read datetime from packed Long as unsigned fixed-len integer - * - * @see DateTimeCodec#fromPackedLong(long, DateTimeZone) - * @param cdi codec buffer input - * @param tz timezone to interpret datetime parts - * @return decoded DateTime using provided timezone - */ - public static DateTime readFromUInt(CodecDataInput cdi, DateTimeZone tz) { - return DateTimeCodec.fromPackedLong(IntegerCodec.readULong(cdi), tz); - } - } - - public static class DateCodec { - - /** - * Encode a UTC Date to a packed long converting to specific timezone - * - * @param date date that need to be encoded. - * @param tz timezone used for converting to localDate - * @return a packed long. - */ - static long toPackedLong(Date date, DateTimeZone tz) { - return toPackedLong(date.getTime(), tz); - } - - static long toPackedLong(long utcMillsTs, DateTimeZone tz) { - LocalDate date = new LocalDate(utcMillsTs, tz); - return toPackedLong(date); - } - - static long toPackedLong(LocalDate date) { - return Codec.DateCodec.toPackedLong( - date.getYear(), date.getMonthOfYear(), date.getDayOfMonth()); - } - - /** - * Encode a date part to a packed long. - * - * @return a packed long. - */ - static long toPackedLong(int year, int month, int day) { - long ymd = (year * 13 + month) << 5 | day; - return ymd << 41; - } - - static LocalDate fromPackedLong(long packed) { - // TODO: As for JDBC behavior, it can be configured to "round" or "toNull" - // for now we didn't pass in session so we do a toNull behavior - if (packed == 0) { - return null; - } - long ymd = packed >> 41; - int day = (int) (ymd & ((1 << 5) - 1)); - long ym = ymd >> 5; - int month = (int) (ym % 13); - int year = (int) (ym / 13); - - return new LocalDate(year, month, day, null); - } - - /** - * Encode Date as packed long converting into specified timezone All timezone conversion should - * be done beforehand - * - * @param cdo encoding output - * @param date value to encode - * @param tz timezone used to converting local time - */ - public static void writeDateFully(CodecDataOutput cdo, Date date, DateTimeZone tz) { - long val = DateCodec.toPackedLong(date, tz); - IntegerCodec.writeULongFully(cdo, val, true); - } - - /** - * Encode Date as packed long converting into specified timezone All timezone conversion should - * be done beforehand The encoded value has no data type flag - * - * @param cdo encoding output - * @param date value to encode - * @param tz timezone used to converting local time - */ - public static void writeDateProto(CodecDataOutput cdo, Date date, DateTimeZone tz) { - long val = DateCodec.toPackedLong(date, tz); - IntegerCodec.writeULong(cdo, val); - } - - /** - * Read date from packed Long encoded as unsigned var-len integer converting into specified - * timezone - * - * @see DateCodec#fromPackedLong(long) - * @param cdi codec buffer input - * @return decoded DateTime using provided timezone - */ - public static LocalDate readFromUVarInt(CodecDataInput cdi) { - return DateCodec.fromPackedLong(IntegerCodec.readUVarLong(cdi)); - } - - /** - * Read date from packed Long as unsigned fixed-len integer - * - * @see DateCodec#fromPackedLong(long) - * @param cdi codec buffer input - * @return decoded DateTime using provided timezone - */ - public static LocalDate readFromUInt(CodecDataInput cdi) { - return DateCodec.fromPackedLong(IntegerCodec.readULong(cdi)); - } - } } diff --git a/src/main/java/org/tikv/event/CacheInvalidateEvent.java b/src/main/java/org/tikv/event/CacheInvalidateEvent.java deleted file mode 100644 index 7d9a35d196..0000000000 --- a/src/main/java/org/tikv/event/CacheInvalidateEvent.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.event; - -import java.io.Serializable; - -public class CacheInvalidateEvent implements Serializable { - public enum CacheType implements Serializable { - REGION_STORE, - REQ_FAILED, - LEADER - } - - private long regionId; - private long storeId; - private boolean invalidateRegion; - private boolean invalidateStore; - private CacheType cacheType; - - public CacheInvalidateEvent( - long regionId, long storeId, boolean updateRegion, boolean updateStore, CacheType type) { - this.regionId = regionId; - this.storeId = storeId; - this.cacheType = type; - if (updateRegion) { - invalidateRegion(); - } - - if (updateStore) { - invalidateStore(); - } - } - - public long getRegionId() { - return regionId; - } - - public long getStoreId() { - return storeId; - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } else if (obj instanceof CacheInvalidateEvent) { - CacheInvalidateEvent event = (CacheInvalidateEvent) obj; - return event.getRegionId() == getRegionId() - && event.getStoreId() == getStoreId() - && event.getCacheType() == getCacheType(); - } - return false; - } - - @Override - public int hashCode() { - int result = 1106; - result += result * 31 + getStoreId(); - result += result * 31 + getRegionId(); - result += result * 31 + getCacheType().name().hashCode(); - return result; - } - - public void invalidateRegion() { - invalidateRegion = true; - } - - public void invalidateStore() { - invalidateStore = true; - } - - public boolean shouldUpdateRegion() { - return invalidateRegion; - } - - public boolean shouldUpdateStore() { - return invalidateStore; - } - - public CacheType getCacheType() { - return cacheType; - } - - @Override - public String toString() { - return String.format("RegionId=%d,StoreId=%d,Type=%s", regionId, storeId, cacheType.name()); - } -} diff --git a/src/main/java/org/tikv/exception/CastingException.java b/src/main/java/org/tikv/exception/CastingException.java deleted file mode 100644 index 80c59a9f08..0000000000 --- a/src/main/java/org/tikv/exception/CastingException.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.exception; - -public class CastingException extends RuntimeException { - public CastingException(Exception e) { - super(e); - } - - public CastingException(String msg) { - super(msg); - } -} diff --git a/src/main/java/org/tikv/exception/DAGRequestException.java b/src/main/java/org/tikv/exception/DAGRequestException.java deleted file mode 100644 index 6e26aa682d..0000000000 --- a/src/main/java/org/tikv/exception/DAGRequestException.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.exception; - -public class DAGRequestException extends RuntimeException { - public DAGRequestException(String msg) { - super(msg); - } -} diff --git a/src/main/java/org/tikv/exception/IgnoreUnsupportedTypeException.java b/src/main/java/org/tikv/exception/IgnoreUnsupportedTypeException.java deleted file mode 100644 index db3e62a942..0000000000 --- a/src/main/java/org/tikv/exception/IgnoreUnsupportedTypeException.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.exception; - -public class IgnoreUnsupportedTypeException extends RuntimeException { - public IgnoreUnsupportedTypeException(String msg) { - super(msg); - } -} diff --git a/src/main/java/org/tikv/exception/RegionTaskException.java b/src/main/java/org/tikv/exception/RegionTaskException.java deleted file mode 100644 index 0994edcddd..0000000000 --- a/src/main/java/org/tikv/exception/RegionTaskException.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2018 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.exception; - -public class RegionTaskException extends RuntimeException { - public RegionTaskException(String msg, Throwable throwable) { - super(msg, throwable); - } -} diff --git a/src/main/java/org/tikv/exception/SelectException.java b/src/main/java/org/tikv/exception/SelectException.java deleted file mode 100644 index f74a654045..0000000000 --- a/src/main/java/org/tikv/exception/SelectException.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.exception; - -import com.pingcap.tidb.tipb.Error; - -public class SelectException extends RuntimeException { - private final Error err; - - public SelectException(Error err, String msg) { - super(msg); - this.err = err; - } - - // TODO: improve this - public SelectException(String msg) { - super(msg); - this.err = null; - } - - public Error getError() { - return err; - } -} diff --git a/src/main/java/org/tikv/exception/TiExpressionException.java b/src/main/java/org/tikv/exception/TiExpressionException.java deleted file mode 100644 index a15bfb20f2..0000000000 --- a/src/main/java/org/tikv/exception/TiExpressionException.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.exception; - -public class TiExpressionException extends RuntimeException { - public TiExpressionException(String msg) { - super(msg); - } - - public TiExpressionException(String msg, Throwable t) { - super(msg, t); - } -} diff --git a/src/main/java/org/tikv/exception/TypeException.java b/src/main/java/org/tikv/exception/TypeException.java deleted file mode 100644 index 6aeb786186..0000000000 --- a/src/main/java/org/tikv/exception/TypeException.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.exception; - -public class TypeException extends RuntimeException { - public TypeException(String msg) { - super(msg); - } - - public TypeException(String msg, Throwable t) { - super(msg, t); - } -} diff --git a/src/main/java/org/tikv/exception/UnsupportedTypeException.java b/src/main/java/org/tikv/exception/UnsupportedTypeException.java deleted file mode 100644 index a293f1bfb1..0000000000 --- a/src/main/java/org/tikv/exception/UnsupportedTypeException.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.exception; - -public class UnsupportedTypeException extends RuntimeException { - public UnsupportedTypeException(String msg) { - super(msg); - } -} diff --git a/src/main/java/org/tikv/expression/AggregateFunction.java b/src/main/java/org/tikv/expression/AggregateFunction.java deleted file mode 100644 index 19e78d1fe9..0000000000 --- a/src/main/java/org/tikv/expression/AggregateFunction.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression; - -import static java.util.Objects.requireNonNull; - -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.Objects; - -public class AggregateFunction implements Expression { - public enum FunctionType { - Sum, - Count, - Min, - Max, - First - } - - private final FunctionType type; - private final Expression argument; - - public static AggregateFunction newCall(FunctionType type, Expression argument) { - return new AggregateFunction(type, argument); - } - - private AggregateFunction(FunctionType type, Expression argument) { - this.type = requireNonNull(type, "function type is null"); - this.argument = requireNonNull(argument, "function argument is null"); - } - - public FunctionType getType() { - return type; - } - - public Expression getArgument() { - return argument; - } - - @Override - public List getChildren() { - return ImmutableList.of(argument); - } - - @Override - public R accept(Visitor visitor, C context) { - return visitor.visit(this, context); - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof AggregateFunction)) { - return false; - } - - AggregateFunction that = (AggregateFunction) other; - return type == that.type && Objects.equals(argument, that.argument); - } - - @Override - public int hashCode() { - return Objects.hash(type, argument); - } - - @Override - public String toString() { - return String.format( - "%s(%s)", getType(), Joiner.on(",").useForNull("NULL").join(getChildren())); - } -} diff --git a/src/main/java/org/tikv/expression/ArithmeticBinaryExpression.java b/src/main/java/org/tikv/expression/ArithmeticBinaryExpression.java deleted file mode 100644 index 2bda9ba9ba..0000000000 --- a/src/main/java/org/tikv/expression/ArithmeticBinaryExpression.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression; - -import static java.util.Objects.requireNonNull; -import static org.tikv.expression.ArithmeticBinaryExpression.Type.*; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.Objects; - -public class ArithmeticBinaryExpression implements Expression { - public enum Type { - PLUS, - MINUS, - MULTIPLY, - DIVIDE, - BIT_AND, - BIT_OR, - BIT_XOR - } - - public static ArithmeticBinaryExpression plus(Expression left, Expression right) { - return new ArithmeticBinaryExpression(PLUS, left, right); - } - - public static ArithmeticBinaryExpression minus(Expression left, Expression right) { - return new ArithmeticBinaryExpression(MINUS, left, right); - } - - public static ArithmeticBinaryExpression multiply(Expression left, Expression right) { - return new ArithmeticBinaryExpression(MULTIPLY, left, right); - } - - public static ArithmeticBinaryExpression divide(Expression left, Expression right) { - return new ArithmeticBinaryExpression(DIVIDE, left, right); - } - - public static ArithmeticBinaryExpression bitAnd(Expression left, Expression right) { - return new ArithmeticBinaryExpression(BIT_AND, left, right); - } - - public static ArithmeticBinaryExpression bitOr(Expression left, Expression right) { - return new ArithmeticBinaryExpression(BIT_OR, left, right); - } - - public static ArithmeticBinaryExpression bitXor(Expression left, Expression right) { - return new ArithmeticBinaryExpression(BIT_XOR, left, right); - } - - private final Expression left; - private final Expression right; - private final Type compType; - - public ArithmeticBinaryExpression(Type type, Expression left, Expression right) { - this.left = requireNonNull(left, "left expression is null"); - this.right = requireNonNull(right, "right expression is null"); - this.compType = requireNonNull(type, "type is null"); - } - - public Expression getLeft() { - return left; - } - - public Expression getRight() { - return right; - } - - public Type getCompType() { - return compType; - } - - @Override - public List getChildren() { - return ImmutableList.of(left, right); - } - - @Override - public R accept(Visitor visitor, C context) { - return visitor.visit(this, context); - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof ArithmeticBinaryExpression)) { - return false; - } - - ArithmeticBinaryExpression that = (ArithmeticBinaryExpression) other; - return (compType == that.compType) - && Objects.equals(left, that.left) - && Objects.equals(right, that.right); - } - - @Override - public int hashCode() { - return Objects.hash(compType, left, right); - } - - @Override - public String toString() { - return String.format("[%s %s %s]", getLeft(), getCompType(), getRight()); - } -} diff --git a/src/main/java/org/tikv/expression/Blacklist.java b/src/main/java/org/tikv/expression/Blacklist.java deleted file mode 100644 index 516c552b89..0000000000 --- a/src/main/java/org/tikv/expression/Blacklist.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.expression; - -import static java.util.Objects.requireNonNull; - -import java.util.HashSet; -import java.util.Set; -import java.util.stream.Collectors; - -public class Blacklist { - private final Set unsupported = new HashSet<>(); - - Blacklist(String string) { - if (string != null) { - String[] some = string.split(","); - for (String one : some) { - String trimmedExprName = one.trim(); - if (!trimmedExprName.isEmpty()) { - unsupported.add(one.trim()); - } - } - } - } - - boolean isUnsupported(String name) { - return unsupported.contains(name); - } - - boolean isUnsupported(Class cls) { - return isUnsupported(requireNonNull(cls).getSimpleName()); - } - - @Override - public String toString() { - return unsupported.stream().collect(Collectors.joining(",")); - } -} diff --git a/src/main/java/org/tikv/expression/ByItem.java b/src/main/java/org/tikv/expression/ByItem.java deleted file mode 100644 index 196f80fc69..0000000000 --- a/src/main/java/org/tikv/expression/ByItem.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression; - -import static com.google.common.base.Preconditions.checkNotNull; - -import java.io.Serializable; -import org.tikv.expression.visitor.ProtoConverter; - -public class ByItem implements Serializable { - private Expression expr; - private boolean desc; - - public static ByItem create(Expression expr, boolean desc) { - return new ByItem(expr, desc); - } - - private ByItem(Expression expr, boolean desc) { - checkNotNull(expr, "Expr cannot be null for ByItem"); - - this.expr = expr; - this.desc = desc; - } - - public com.pingcap.tidb.tipb.ByItem toProto(Object context) { - com.pingcap.tidb.tipb.ByItem.Builder builder = com.pingcap.tidb.tipb.ByItem.newBuilder(); - return builder.setExpr(ProtoConverter.toProto(expr, context)).setDesc(desc).build(); - } - - public Expression getExpr() { - return expr; - } - - public boolean isDesc() { - return desc; - } - - @Override - public String toString() { - return String.format("[%s %s]", expr.toString(), desc ? "DESC" : "ASC"); - } -} diff --git a/src/main/java/org/tikv/expression/ColumnRef.java b/src/main/java/org/tikv/expression/ColumnRef.java deleted file mode 100644 index fe267e9198..0000000000 --- a/src/main/java/org/tikv/expression/ColumnRef.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.Objects; -import org.tikv.exception.TiClientInternalException; -import org.tikv.exception.TiExpressionException; -import org.tikv.meta.TiColumnInfo; -import org.tikv.meta.TiTableInfo; -import org.tikv.types.DataType; - -public class ColumnRef implements Expression { - public static ColumnRef create(String name, TiTableInfo table) { - for (TiColumnInfo columnInfo : table.getColumns()) { - if (columnInfo.matchName(name)) { - return new ColumnRef(columnInfo.getName(), columnInfo, table); - } - } - throw new TiExpressionException( - String.format("Column name %s not found in table %s", name, table)); - } - - public static ColumnRef create(String name) { - return new ColumnRef(name); - } - - private final String name; - - private TiColumnInfo columnInfo; - private TiTableInfo tableInfo; - - public ColumnRef(String name) { - this.name = name; - } - - public ColumnRef(String name, TiColumnInfo columnInfo, TiTableInfo tableInfo) { - this.name = name; - this.columnInfo = columnInfo; - this.tableInfo = tableInfo; - } - - public String getName() { - return name; - } - - public void resolve(TiTableInfo table) { - TiColumnInfo columnInfo = null; - for (TiColumnInfo col : table.getColumns()) { - if (col.matchName(name)) { - columnInfo = col; - break; - } - } - if (columnInfo == null) { - throw new TiExpressionException( - String.format("No Matching column %s from table %s", name, table.getName())); - } - - if (columnInfo.getId() == 0) { - throw new TiExpressionException("Zero Id is not a referable column id"); - } - - this.tableInfo = table; - this.columnInfo = columnInfo; - } - - public TiColumnInfo getColumnInfo() { - if (columnInfo == null) { - throw new TiClientInternalException(String.format("ColumnRef [%s] is unbound", name)); - } - return columnInfo; - } - - public DataType getType() { - return getColumnInfo().getType(); - } - - public TiTableInfo getTableInfo() { - return tableInfo; - } - - public boolean isResolved() { - return tableInfo != null && columnInfo != null; - } - - @Override - public boolean equals(Object another) { - if (this == another) { - return true; - } - - if (another instanceof ColumnRef) { - ColumnRef that = (ColumnRef) another; - if (isResolved() && that.isResolved()) { - return Objects.equals(columnInfo, that.columnInfo) - && Objects.equals(tableInfo, that.tableInfo); - } else { - return name.equalsIgnoreCase(that.name); - } - } else { - return false; - } - } - - @Override - public int hashCode() { - if (isResolved()) { - return Objects.hash(tableInfo, columnInfo); - } else { - return Objects.hashCode(name); - } - } - - @Override - public String toString() { - return String.format("[%s]", getName()); - } - - @Override - public List getChildren() { - return ImmutableList.of(); - } - - @Override - public R accept(Visitor visitor, C context) { - return visitor.visit(this, context); - } -} diff --git a/src/main/java/org/tikv/expression/ComparisonBinaryExpression.java b/src/main/java/org/tikv/expression/ComparisonBinaryExpression.java deleted file mode 100644 index 58245f8183..0000000000 --- a/src/main/java/org/tikv/expression/ComparisonBinaryExpression.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; -import static org.tikv.expression.ComparisonBinaryExpression.Type.*; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import org.tikv.exception.TiExpressionException; -import org.tikv.key.TypedKey; -import org.tikv.types.DataType; - -public class ComparisonBinaryExpression implements Expression { - public enum Type { - EQUAL, - NOT_EQUAL, - LESS_THAN, - LESS_EQUAL, - GREATER_THAN, - GREATER_EQUAL - } - - public static ComparisonBinaryExpression equal(Expression left, Expression right) { - return new ComparisonBinaryExpression(EQUAL, left, right); - } - - public static ComparisonBinaryExpression notEqual(Expression left, Expression right) { - return new ComparisonBinaryExpression(NOT_EQUAL, left, right); - } - - public static ComparisonBinaryExpression lessThan(Expression left, Expression right) { - return new ComparisonBinaryExpression(LESS_THAN, left, right); - } - - public static ComparisonBinaryExpression lessEqual(Expression left, Expression right) { - return new ComparisonBinaryExpression(LESS_EQUAL, left, right); - } - - public static ComparisonBinaryExpression greaterThan(Expression left, Expression right) { - return new ComparisonBinaryExpression(GREATER_THAN, left, right); - } - - public static ComparisonBinaryExpression greaterEqual(Expression left, Expression right) { - return new ComparisonBinaryExpression(GREATER_EQUAL, left, right); - } - - public static class NormalizedPredicate { - private final ComparisonBinaryExpression pred; - private TypedKey key; - - NormalizedPredicate(ComparisonBinaryExpression pred) { - checkArgument(pred.getLeft() instanceof ColumnRef); - checkArgument(pred.getRight() instanceof Constant); - this.pred = pred; - } - - public ColumnRef getColumnRef() { - return (ColumnRef) pred.getLeft(); - } - - public Constant getValue() { - return (Constant) pred.getRight(); - } - - public Type getType() { - return pred.getComparisonType(); - } - - public TypedKey getTypedLiteral() { - return getTypedLiteral(DataType.UNSPECIFIED_LEN); - } - - public TypedKey getTypedLiteral(int prefixLength) { - if (key == null) { - key = TypedKey.toTypedKey(getValue().getValue(), getColumnRef().getType(), prefixLength); - } - return key; - } - } - - private final Expression left; - private final Expression right; - private final Type compType; - private transient Optional normalizedPredicate; - - public ComparisonBinaryExpression(Type type, Expression left, Expression right) { - this.left = requireNonNull(left, "left expression is null"); - this.right = requireNonNull(right, "right expression is null"); - this.compType = requireNonNull(type, "type is null"); - } - - @Override - public List getChildren() { - return ImmutableList.of(left, right); - } - - @Override - public R accept(Visitor visitor, C context) { - return visitor.visit(this, context); - } - - public Expression getLeft() { - return left; - } - - public Expression getRight() { - return right; - } - - public Type getComparisonType() { - return compType; - } - - public NormalizedPredicate normalize() { - if (normalizedPredicate != null) { - return normalizedPredicate.orElseGet(null); - } - if (getLeft() instanceof Constant && getRight() instanceof ColumnRef) { - Constant left = (Constant) getLeft(); - ColumnRef right = (ColumnRef) getRight(); - Type newType; - switch (getComparisonType()) { - case EQUAL: - newType = EQUAL; - break; - case LESS_EQUAL: - newType = GREATER_EQUAL; - break; - case LESS_THAN: - newType = GREATER_THAN; - break; - case GREATER_EQUAL: - newType = LESS_EQUAL; - break; - case GREATER_THAN: - newType = LESS_THAN; - break; - case NOT_EQUAL: - newType = NOT_EQUAL; - break; - default: - throw new TiExpressionException( - String.format( - "PredicateNormalizer is not able to process type %s", getComparisonType())); - } - ComparisonBinaryExpression newExpression = - new ComparisonBinaryExpression(newType, right, left); - normalizedPredicate = Optional.of(new NormalizedPredicate(newExpression)); - return normalizedPredicate.get(); - } else if (getRight() instanceof Constant && getLeft() instanceof ColumnRef) { - normalizedPredicate = Optional.of(new NormalizedPredicate(this)); - return normalizedPredicate.get(); - } else { - return null; - } - } - - @Override - public String toString() { - return String.format("[%s %s %s]", getLeft(), getComparisonType(), getRight()); - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof ComparisonBinaryExpression)) { - return false; - } - - ComparisonBinaryExpression that = (ComparisonBinaryExpression) other; - return (compType == that.compType) - && Objects.equals(left, that.left) - && Objects.equals(right, that.right); - } - - @Override - public int hashCode() { - return Objects.hash(compType, left, right); - } -} diff --git a/src/main/java/org/tikv/expression/Constant.java b/src/main/java/org/tikv/expression/Constant.java deleted file mode 100644 index f1fd810b24..0000000000 --- a/src/main/java/org/tikv/expression/Constant.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression; - -import com.google.common.collect.ImmutableList; -import java.math.BigDecimal; -import java.sql.Date; -import java.sql.Timestamp; -import java.util.List; -import java.util.Objects; -import org.joda.time.DateTime; -import org.tikv.exception.TiExpressionException; -import org.tikv.types.*; - -// Refactor needed. -// Refer to https://github.com/pingcap/tipb/blob/master/go-tipb/expression.pb.go -// TODO: This might need a refactor to accept an DataType? -public class Constant implements Expression { - private final Object value; - private DataType type; - - public static Constant create(Object value, DataType type) { - return new Constant(value, type); - } - - public static Constant create(Object value) { - return new Constant(value, null); - } - - public Constant(Object value, DataType type) { - this.value = value; - this.type = (type == null && value != null) ? getDefaultType(value) : type; - } - - protected static boolean isIntegerType(Object value) { - return value instanceof Long - || value instanceof Integer - || value instanceof Short - || value instanceof Byte; - } - - private static DataType getDefaultType(Object value) { - if (value == null) { - throw new TiExpressionException("NULL constant has no type"); - } else if (isIntegerType(value)) { - return IntegerType.BIGINT; - } else if (value instanceof String) { - return StringType.VARCHAR; - } else if (value instanceof Float) { - return RealType.FLOAT; - } else if (value instanceof Double) { - return RealType.DOUBLE; - } else if (value instanceof BigDecimal) { - return DecimalType.DECIMAL; - } else if (value instanceof DateTime) { - return DateTimeType.DATETIME; - } else if (value instanceof Date) { - return DateType.DATE; - } else if (value instanceof Timestamp) { - return TimestampType.TIMESTAMP; - } else if (value instanceof byte[]) { - return BytesType.TEXT; - } else { - throw new TiExpressionException( - "Constant type not supported:" + value.getClass().getSimpleName()); - } - } - - public void setType(DataType type) { - this.type = type; - } - - public Object getValue() { - return value; - } - - public DataType getType() { - return type; - } - - @Override - public String toString() { - if (value == null) { - return "null"; - } - if (value instanceof String) { - return String.format("\"%s\"", value); - } - return value.toString(); - } - - @Override - public boolean equals(Object other) { - if (other instanceof Constant) { - return Objects.equals(value, ((Constant) other).value); - } - return false; - } - - @Override - public int hashCode() { - return Objects.hashCode(value); - } - - @Override - public List getChildren() { - return ImmutableList.of(); - } - - @Override - public R accept(Visitor visitor, C context) { - return visitor.visit(this, context); - } -} diff --git a/src/main/java/org/tikv/expression/Expression.java b/src/main/java/org/tikv/expression/Expression.java deleted file mode 100644 index 12d0f3021c..0000000000 --- a/src/main/java/org/tikv/expression/Expression.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression; - -import java.io.Serializable; -import java.util.List; - -public interface Expression extends Serializable { - List getChildren(); - - R accept(Visitor visitor, C context); -} diff --git a/src/main/java/org/tikv/expression/ExpressionBlacklist.java b/src/main/java/org/tikv/expression/ExpressionBlacklist.java deleted file mode 100644 index fa54254948..0000000000 --- a/src/main/java/org/tikv/expression/ExpressionBlacklist.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression; - -public class ExpressionBlacklist extends Blacklist { - - public ExpressionBlacklist(String exprsString) { - super(exprsString); - } - - public boolean isUnsupportedPushdownExpr(Class cls) { - return isUnsupported(cls); - } -} diff --git a/src/main/java/org/tikv/expression/IsNull.java b/src/main/java/org/tikv/expression/IsNull.java deleted file mode 100644 index f407244a4c..0000000000 --- a/src/main/java/org/tikv/expression/IsNull.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression; - -import static java.util.Objects.requireNonNull; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.Objects; - -public class IsNull implements Expression { - private Expression expression; - - public IsNull(Expression expression) { - this.expression = requireNonNull(expression, "expression is null"); - } - - public Expression getExpression() { - return expression; - } - - @Override - public List getChildren() { - return ImmutableList.of(expression); - } - - @Override - public R accept(Visitor visitor, C context) { - return visitor.visit(this, context); - } - - @Override - public String toString() { - return String.format("IsNull(%s)", getExpression()); - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof IsNull)) { - return false; - } - - IsNull that = (IsNull) other; - return Objects.equals(expression, that.expression); - } - - @Override - public int hashCode() { - return Objects.hashCode(expression); - } -} diff --git a/src/main/java/org/tikv/expression/LogicalBinaryExpression.java b/src/main/java/org/tikv/expression/LogicalBinaryExpression.java deleted file mode 100644 index 79c8b7d108..0000000000 --- a/src/main/java/org/tikv/expression/LogicalBinaryExpression.java +++ /dev/null @@ -1,87 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: expression.proto - -package org.tikv.expression; - -import static java.util.Objects.requireNonNull; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.Objects; - -public class LogicalBinaryExpression implements Expression { - public enum Type { - AND, - OR, - XOR - } - - public static LogicalBinaryExpression and(Expression left, Expression right) { - return new LogicalBinaryExpression(Type.AND, left, right); - } - - public static LogicalBinaryExpression or(Expression left, Expression right) { - return new LogicalBinaryExpression(Type.OR, left, right); - } - - public static LogicalBinaryExpression xor(Expression left, Expression right) { - return new LogicalBinaryExpression(Type.XOR, left, right); - } - - public LogicalBinaryExpression(Type type, Expression left, Expression right) { - this.left = requireNonNull(left, "left expression is null"); - this.right = requireNonNull(right, "right expression is null"); - this.compType = requireNonNull(type, "type is null"); - } - - @Override - public List getChildren() { - return ImmutableList.of(getLeft(), getRight()); - } - - @Override - public R accept(Visitor visitor, C context) { - return visitor.visit(this, context); - } - - public Expression getLeft() { - return left; - } - - public Expression getRight() { - return right; - } - - public Type getCompType() { - return compType; - } - - private final Expression left; - private final Expression right; - private final Type compType; - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof LogicalBinaryExpression)) { - return false; - } - - LogicalBinaryExpression that = (LogicalBinaryExpression) other; - return (compType == that.compType) - && Objects.equals(left, that.left) - && Objects.equals(right, that.right); - } - - @Override - public int hashCode() { - return Objects.hash(compType, left, right); - } - - @Override - public String toString() { - return String.format("[%s %s %s]", getLeft(), getCompType(), getRight()); - } -} diff --git a/src/main/java/org/tikv/expression/Not.java b/src/main/java/org/tikv/expression/Not.java deleted file mode 100644 index cab66a204a..0000000000 --- a/src/main/java/org/tikv/expression/Not.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression; - -import static java.util.Objects.requireNonNull; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.Objects; - -public class Not implements Expression { - - public static Not not(Expression expression) { - return new Not(expression); - } - - private Expression expression; - - public Not(Expression expression) { - this.expression = requireNonNull(expression, "expression is null"); - } - - public Expression getExpression() { - return expression; - } - - @Override - public List getChildren() { - return ImmutableList.of(expression); - } - - @Override - public R accept(Visitor visitor, C context) { - return visitor.visit(this, context); - } - - @Override - public String toString() { - return String.format("Not(%s)", getExpression()); - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof Not)) { - return false; - } - - Not that = (Not) other; - return Objects.equals(expression, that.expression); - } - - @Override - public int hashCode() { - return Objects.hashCode(expression); - } -} diff --git a/src/main/java/org/tikv/expression/StringRegExpression.java b/src/main/java/org/tikv/expression/StringRegExpression.java deleted file mode 100644 index dac5cc09bb..0000000000 --- a/src/main/java/org/tikv/expression/StringRegExpression.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression; - -import static java.util.Objects.requireNonNull; -import static org.tikv.expression.StringRegExpression.Type.*; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.Objects; -import org.tikv.key.TypedKey; -import org.tikv.types.DataType; -import org.tikv.types.IntegerType; - -public class StringRegExpression implements Expression { - public enum Type { - STARTS_WITH, - CONTAINS, - ENDS_WITH, - LIKE - } - - public static StringRegExpression startsWith(Expression left, Expression right) { - Expression reg = - Constant.create(((Constant) right).getValue() + "%", ((Constant) right).getType()); - return new StringRegExpression(STARTS_WITH, left, right, reg); - } - - public static StringRegExpression contains(Expression left, Expression right) { - Expression reg = - Constant.create("%" + ((Constant) right).getValue() + "%", ((Constant) right).getType()); - return new StringRegExpression(CONTAINS, left, right, reg); - } - - public static StringRegExpression endsWith(Expression left, Expression right) { - Expression reg = - Constant.create("%" + ((Constant) right).getValue(), ((Constant) right).getType()); - return new StringRegExpression(ENDS_WITH, left, right, reg); - } - - public static StringRegExpression like(Expression left, Expression right) { - return new StringRegExpression(LIKE, left, right, right); - } - - private transient TypedKey key; - - public ColumnRef getColumnRef() { - return (ColumnRef) getLeft(); - } - - public Constant getValue() { - return (Constant) getRight(); - } - - public TypedKey getTypedLiteral() { - return getTypedLiteral(DataType.UNSPECIFIED_LEN); - } - - public TypedKey getTypedLiteral(int prefixLength) { - if (key == null) { - key = TypedKey.toTypedKey(getValue().getValue(), getColumnRef().getType(), prefixLength); - } - return key; - } - - private final Expression left; - private final Expression right; - private final Expression reg; - private final Type regType; - - public StringRegExpression(Type type, Expression left, Expression right, Expression reg) { - this.left = requireNonNull(left, "left expression is null"); - this.right = requireNonNull(right, "right expression is null"); - this.regType = requireNonNull(type, "type is null"); - this.reg = requireNonNull(reg, "reg string is null"); - } - - @Override - public List getChildren() { - // For LIKE statement, an extra ESCAPE parameter is required as the third parameter for - // ScalarFunc. - // However in Spark ESCAPE is not supported so we simply set this value to zero. - return ImmutableList.of(left, reg, Constant.create(0, IntegerType.BIGINT)); - } - - @Override - public R accept(Visitor visitor, C context) { - return visitor.visit(this, context); - } - - public Expression getLeft() { - return left; - } - - public Expression getRight() { - return right; - } - - public Type getRegType() { - return regType; - } - - public Expression getReg() { - return reg; - } - - @Override - public String toString() { - return String.format("[%s %s %s reg: %s]", getLeft(), getRegType(), getRight(), getReg()); - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof StringRegExpression)) { - return false; - } - - StringRegExpression that = (StringRegExpression) other; - return (regType == that.regType) - && Objects.equals(left, that.left) - && Objects.equals(left, that.right) - && Objects.equals(reg, that.reg); - } - - @Override - public int hashCode() { - return Objects.hash(regType, left, right, reg); - } -} diff --git a/src/main/java/org/tikv/expression/TypeBlacklist.java b/src/main/java/org/tikv/expression/TypeBlacklist.java deleted file mode 100644 index 9bac9a3d0f..0000000000 --- a/src/main/java/org/tikv/expression/TypeBlacklist.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.expression; - -import static org.tikv.types.MySQLType.*; - -import java.util.HashMap; -import java.util.Map; -import org.tikv.types.MySQLType; - -public class TypeBlacklist extends Blacklist { - private static final Map typeToMySQLMap = initialTypeMap(); - - private static HashMap initialTypeMap() { - HashMap map = new HashMap<>(); - map.put(TypeDecimal, "decimal"); - map.put(TypeTiny, "tinyint"); - map.put(TypeShort, "smallint"); - map.put(TypeLong, "int"); - map.put(TypeFloat, "float"); - map.put(TypeDouble, "double"); - map.put(TypeNull, "null"); - map.put(TypeTimestamp, "timestamp"); - map.put(TypeLonglong, "bigint"); - map.put(TypeInt24, "mediumint"); - map.put(TypeDate, "date"); - map.put(TypeDuration, "time"); - map.put(TypeDatetime, "datetime"); - map.put(TypeYear, "year"); - map.put(TypeNewDate, "date"); - map.put(TypeVarchar, "varchar"); - map.put(TypeJSON, "json"); - map.put(TypeNewDecimal, "decimal"); - map.put(TypeEnum, "enum"); - map.put(TypeSet, "set"); - map.put(TypeTinyBlob, "tinytext"); - map.put(TypeMediumBlob, "mediumtext"); - map.put(TypeLongBlob, "longtext"); - map.put(TypeBlob, "text"); - map.put(TypeVarString, "varString"); - map.put(TypeString, "string"); - return map; - } - - public TypeBlacklist(String typesString) { - super(typesString); - } - - public boolean isUnsupportedType(MySQLType sqlType) { - return isUnsupported(typeToMySQLMap.getOrDefault(sqlType, "")); - } -} diff --git a/src/main/java/org/tikv/expression/Visitor.java b/src/main/java/org/tikv/expression/Visitor.java deleted file mode 100644 index cc98219b33..0000000000 --- a/src/main/java/org/tikv/expression/Visitor.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.expression; - -public abstract class Visitor { - protected abstract R visit(ColumnRef node, C context); - - protected abstract R visit(ComparisonBinaryExpression node, C context); - - protected abstract R visit(StringRegExpression node, C context); - - protected abstract R visit(ArithmeticBinaryExpression node, C context); - - protected abstract R visit(LogicalBinaryExpression node, C context); - - protected abstract R visit(Constant node, C context); - - protected abstract R visit(AggregateFunction node, C context); - - protected abstract R visit(IsNull node, C context); - - protected abstract R visit(Not node, C context); -} diff --git a/src/main/java/org/tikv/expression/visitor/ColumnMatcher.java b/src/main/java/org/tikv/expression/visitor/ColumnMatcher.java deleted file mode 100644 index 5894c1d60e..0000000000 --- a/src/main/java/org/tikv/expression/visitor/ColumnMatcher.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2018 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression.visitor; - -import org.tikv.expression.ColumnRef; -import org.tikv.expression.Expression; - -public class ColumnMatcher extends DefaultVisitor { - private final ColumnRef columnRef; - - private ColumnMatcher(ColumnRef exp) { - this.columnRef = exp; - } - - public static Boolean match(ColumnRef col, Expression expression) { - ColumnMatcher matcher = new ColumnMatcher(col); - return expression.accept(matcher, null); - } - - @Override - protected Boolean process(Expression node, Void context) { - return false; - } - - @Override - protected Boolean visit(ColumnRef node, Void context) { - return node.getColumnInfo().matchName(columnRef.getName()); - } -} diff --git a/src/main/java/org/tikv/expression/visitor/DefaultVisitor.java b/src/main/java/org/tikv/expression/visitor/DefaultVisitor.java deleted file mode 100644 index 516f34d734..0000000000 --- a/src/main/java/org/tikv/expression/visitor/DefaultVisitor.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression.visitor; - -import org.tikv.expression.*; - -public class DefaultVisitor extends Visitor { - protected R process(Expression node, C context) { - for (Expression expr : node.getChildren()) { - expr.accept(this, context); - } - return null; - } - - @Override - protected R visit(ColumnRef node, C context) { - return process(node, context); - } - - @Override - protected R visit(ComparisonBinaryExpression node, C context) { - return process(node, context); - } - - @Override - protected R visit(StringRegExpression node, C context) { - return process(node, context); - } - - @Override - protected R visit(ArithmeticBinaryExpression node, C context) { - return process(node, context); - } - - @Override - protected R visit(LogicalBinaryExpression node, C context) { - return process(node, context); - } - - @Override - protected R visit(Constant node, C context) { - return process(node, context); - } - - @Override - protected R visit(AggregateFunction node, C context) { - return process(node, context); - } - - @Override - protected R visit(IsNull node, C context) { - return process(node, context); - } - - @Override - protected R visit(Not node, C context) { - return process(node, context); - } -} diff --git a/src/main/java/org/tikv/expression/visitor/ExpressionTypeCoercer.java b/src/main/java/org/tikv/expression/visitor/ExpressionTypeCoercer.java deleted file mode 100644 index 82c7789355..0000000000 --- a/src/main/java/org/tikv/expression/visitor/ExpressionTypeCoercer.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression.visitor; - -import static java.util.Objects.requireNonNull; - -import java.util.IdentityHashMap; -import java.util.List; -import org.tikv.exception.TiExpressionException; -import org.tikv.expression.*; -import org.tikv.expression.AggregateFunction.FunctionType; -import org.tikv.types.DataType; -import org.tikv.types.DecimalType; -import org.tikv.types.IntegerType; -import org.tikv.types.RealType; -import org.tikv.util.Pair; - -/** - * Validate and infer expression type Collected results are returned getTypeMap For now we don't do - * any type promotion and only coerce from left to right. - */ -public class ExpressionTypeCoercer extends Visitor, DataType> { - private final IdentityHashMap typeMap = new IdentityHashMap<>(); - private static final double MAX_CREDIBILITY = 1.0; - private static final double MIN_CREDIBILITY = 0.1; - private static final double COLUMN_REF_CRED = MAX_CREDIBILITY; - private static final double CONSTANT_CRED = MIN_CREDIBILITY; - private static final double LOGICAL_OP_CRED = MAX_CREDIBILITY; - private static final double COMPARISON_OP_CRED = MAX_CREDIBILITY; - private static final double SRING_REG_OP_CRED = MAX_CREDIBILITY; - private static final double FUNCTION_CRED = MAX_CREDIBILITY; - private static final double ISNULL_CRED = MAX_CREDIBILITY; - private static final double NOT_CRED = MAX_CREDIBILITY; - - public IdentityHashMap getTypeMap() { - return typeMap; - } - - public static DataType inferType(Expression expression) { - ExpressionTypeCoercer inf = new ExpressionTypeCoercer(); - return inf.infer(expression); - } - - public DataType infer(Expression expression) { - requireNonNull(expression, "expression is null"); - return expression.accept(this, null).first; - } - - public void infer(List expressions) { - requireNonNull(expressions, "expressions is null"); - expressions.forEach(expr -> expr.accept(this, null)); - } - - @Override - protected Pair visit(ColumnRef node, DataType targetType) { - DataType type = node.getType(); - if (targetType != null && !targetType.equals(type)) { - throw new TiExpressionException(String.format("Column %s cannot be %s", node, targetType)); - } - typeMap.put(node, type); - return Pair.create(type, COLUMN_REF_CRED); - } - - // Try to coerceType if needed - // A column reference is source of coerce and constant is the subject to coerce - // targetType null means no coerce needed from parent and choose the highest credibility result - protected Pair coerceType(DataType targetType, Expression... nodes) { - if (nodes.length == 0) { - throw new TiExpressionException("failed to verify empty node list"); - } - if (targetType == null) { - Pair baseline = nodes[0].accept(this, null); - for (int i = 1; i < nodes.length; i++) { - Pair current = nodes[i].accept(this, null); - if (current.second > baseline.second) { - baseline = current; - } - } - for (Expression node : nodes) { - node.accept(this, baseline.first); - } - return baseline; - } else { - double credibility = -1; - for (Expression node : nodes) { - Pair result = node.accept(this, targetType); - if (result.second > credibility) { - credibility = result.second; - } - } - return Pair.create(targetType, credibility); - } - } - - @Override - protected Pair visit(ComparisonBinaryExpression node, DataType targetType) { - if (targetType != null && !targetType.equals(IntegerType.BOOLEAN)) { - throw new TiExpressionException(String.format("Comparison result cannot be %s", targetType)); - } - if (!typeMap.containsKey(node)) { - coerceType(null, node.getLeft(), node.getRight()); - typeMap.put(node, IntegerType.BOOLEAN); - } - return Pair.create(IntegerType.BOOLEAN, COMPARISON_OP_CRED); - } - - @Override - protected Pair visit(StringRegExpression node, DataType targetType) { - if (targetType != null && !targetType.equals(IntegerType.BOOLEAN)) { - throw new TiExpressionException(String.format("Comparison result cannot be %s", targetType)); - } - if (!typeMap.containsKey(node)) { - coerceType(null, node.getLeft(), node.getRight()); - typeMap.put(node, IntegerType.BOOLEAN); - } - return Pair.create(IntegerType.BOOLEAN, SRING_REG_OP_CRED); - } - - @Override - protected Pair visit(ArithmeticBinaryExpression node, DataType targetType) { - Pair result = coerceType(targetType, node.getLeft(), node.getRight()); - typeMap.put(node, result.first); - return result; - } - - @Override - protected Pair visit(LogicalBinaryExpression node, DataType targetType) { - if (targetType != null && !targetType.equals(IntegerType.BOOLEAN)) { - throw new TiExpressionException(String.format("Comparison result cannot be %s", targetType)); - } - if (!typeMap.containsKey(node)) { - coerceType(null, node.getLeft(), node.getRight()); - typeMap.put(node, IntegerType.BOOLEAN); - } - return Pair.create(IntegerType.BOOLEAN, LOGICAL_OP_CRED); - } - - @Override - protected Pair visit(Constant node, DataType targetType) { - if (targetType == null) { - return Pair.create(node.getType(), CONSTANT_CRED); - } else { - node.setType(targetType); - typeMap.put(node, targetType); - return Pair.create(targetType, CONSTANT_CRED); - } - } - - @Override - protected Pair visit(AggregateFunction node, DataType targetType) { - FunctionType fType = node.getType(); - coerceType(null, node.getArgument()); - switch (fType) { - case Count: - { - if (targetType != null && targetType.equals(IntegerType.BIGINT)) { - throw new TiExpressionException(String.format("Count cannot be %s", targetType)); - } - typeMap.put(node, IntegerType.BIGINT); - return Pair.create(targetType, FUNCTION_CRED); - } - case Sum: - { - if (targetType != null && targetType.equals(DecimalType.DECIMAL)) { - throw new TiExpressionException(String.format("Sum cannot be %s", targetType)); - } - DataType colType = node.getArgument().accept(this, null).first; - if (colType instanceof RealType) { - typeMap.put(node, RealType.DOUBLE); - } else { - typeMap.put(node, DecimalType.DECIMAL); - } - return Pair.create(targetType, FUNCTION_CRED); - } - case First: - case Max: - case Min: - { - Pair result = coerceType(targetType, node.getArgument()); - typeMap.put(node, result.first); - return result; - } - default: - throw new TiExpressionException(String.format("Unknown function %s", fType)); - } - } - - @Override - protected Pair visit(IsNull node, DataType targetType) { - if (targetType != null && !targetType.equals(IntegerType.BOOLEAN)) { - throw new TiExpressionException(String.format("IsNull result cannot be %s", targetType)); - } - if (!typeMap.containsKey(node)) { - coerceType(null, node.getExpression()); - typeMap.put(node, IntegerType.BOOLEAN); - } - return Pair.create(IntegerType.BOOLEAN, ISNULL_CRED); - } - - @Override - protected Pair visit(Not node, DataType targetType) { - if (targetType != null && !targetType.equals(IntegerType.BOOLEAN)) { - throw new TiExpressionException(String.format("Not result cannot be %s", targetType)); - } - if (!typeMap.containsKey(node)) { - coerceType(null, node.getExpression()); - typeMap.put(node, IntegerType.BOOLEAN); - } - return Pair.create(IntegerType.BOOLEAN, NOT_CRED); - } -} diff --git a/src/main/java/org/tikv/expression/visitor/IndexMatcher.java b/src/main/java/org/tikv/expression/visitor/IndexMatcher.java deleted file mode 100644 index d97362379a..0000000000 --- a/src/main/java/org/tikv/expression/visitor/IndexMatcher.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression.visitor; - -import static java.util.Objects.requireNonNull; - -import org.tikv.expression.*; -import org.tikv.expression.ComparisonBinaryExpression.NormalizedPredicate; -import org.tikv.meta.TiIndexColumn; - -/** - * Test if a predicate matches and index column entirely and can be convert to index related ranges - * If a predicate matches only partially, it returns false - */ -public class IndexMatcher extends DefaultVisitor { - private final boolean matchEqualTestOnly; - private final TiIndexColumn indexColumn; - - private IndexMatcher(TiIndexColumn indexColumn, boolean matchEqualTestOnly) { - this.matchEqualTestOnly = matchEqualTestOnly; - this.indexColumn = requireNonNull(indexColumn, "index column is null"); - } - - public static IndexMatcher equalOnlyMatcher(TiIndexColumn indexColumn) { - return new IndexMatcher(indexColumn, true); - } - - public static IndexMatcher matcher(TiIndexColumn indexColumn) { - return new IndexMatcher(indexColumn, false); - } - - public boolean match(Expression expression) { - return expression.accept(this, null); - } - - @Override - protected Boolean process(Expression node, Void context) { - return false; - } - - @Override - protected Boolean visit(ColumnRef node, Void context) { - String indexColumnName = indexColumn.getName(); - return node.getColumnInfo().matchName(indexColumnName); - } - - @Override - protected Boolean visit(ComparisonBinaryExpression node, Void context) { - switch (node.getComparisonType()) { - case LESS_THAN: - case LESS_EQUAL: - case GREATER_THAN: - case GREATER_EQUAL: - case NOT_EQUAL: - if (matchEqualTestOnly) { - return false; - } - case EQUAL: - NormalizedPredicate predicate = node.normalize(); - if (predicate == null) { - return false; - } - return predicate.getColumnRef().accept(this, context); - default: - return false; - } - } - - @Override - protected Boolean visit(StringRegExpression node, Void context) { - switch (node.getRegType()) { - // If the predicate is StartsWith(col, 'a'), this predicate - // indicates a range of ['a', +∞) which can be used by index scan - case STARTS_WITH: - if (matchEqualTestOnly) { - return false; - } - return node.getLeft().accept(this, context); - default: - return false; - } - } - - @Override - protected Boolean visit(LogicalBinaryExpression node, Void context) { - switch (node.getCompType()) { - case AND: - if (matchEqualTestOnly) { - return false; - } - case OR: - case XOR: - return node.getLeft().accept(this, context) && node.getRight().accept(this, context); - default: - return false; - } - } -} diff --git a/src/main/java/org/tikv/expression/visitor/IndexRangeBuilder.java b/src/main/java/org/tikv/expression/visitor/IndexRangeBuilder.java deleted file mode 100644 index 1c509a1819..0000000000 --- a/src/main/java/org/tikv/expression/visitor/IndexRangeBuilder.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression.visitor; - -import com.google.common.collect.Range; -import com.google.common.collect.RangeSet; -import com.google.common.collect.TreeRangeSet; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import org.tikv.exception.TiExpressionException; -import org.tikv.expression.*; -import org.tikv.expression.ComparisonBinaryExpression.NormalizedPredicate; -import org.tikv.key.TypedKey; -import org.tikv.meta.TiIndexColumn; -import org.tikv.meta.TiIndexInfo; -import org.tikv.meta.TiTableInfo; -import org.tikv.types.DataType; - -public class IndexRangeBuilder extends DefaultVisitor, Void> { - - private final Map lengths; // length of corresponding ColumnRef - - public IndexRangeBuilder(TiTableInfo table, TiIndexInfo index) { - Map result = new HashMap<>(); - if (table != null && index != null) { - for (TiIndexColumn indexColumn : index.getIndexColumns()) { - ColumnRef columnRef = ColumnRef.create(indexColumn.getName(), table); - result.put(columnRef, (int) indexColumn.getLength()); - } - } - this.lengths = result; - } - - public Set> buildRange(Expression predicate) { - Objects.requireNonNull(predicate, "predicate is null"); - return predicate.accept(this, null).asRanges(); - } - - private static void throwOnError(Expression node) { - final String errorFormat = "Unsupported conversion to Range: %s"; - throw new TiExpressionException(String.format(errorFormat, node)); - } - - protected RangeSet process(Expression node, Void context) { - throwOnError(node); - return null; - } - - @Override - protected RangeSet visit(LogicalBinaryExpression node, Void context) { - RangeSet leftRanges = node.getLeft().accept(this, context); - RangeSet rightRanges = node.getRight().accept(this, context); - switch (node.getCompType()) { - case AND: - for (Range range : leftRanges.asRanges()) { - rightRanges = rightRanges.subRangeSet(range); - } - break; - case OR: - rightRanges.addAll(leftRanges); - break; - case XOR: - // AND - RangeSet intersection = rightRanges; - for (Range range : leftRanges.asRanges()) { - intersection = intersection.subRangeSet(range); - } - // full set - rightRanges.addAll(leftRanges); - rightRanges.removeAll(intersection); - break; - default: - throwOnError(node); - } - return rightRanges; - } - - @Override - protected RangeSet visit(ComparisonBinaryExpression node, Void context) { - NormalizedPredicate predicate = node.normalize(); - if (predicate == null) { - throwOnError(node); - } - // In order to match a prefix index, we have to cut the literal by prefix length. - // e.g., for table t: - // CREATE TABLE `t` { - // `b` VARCHAR(10) DEFAULT NULL, - // KEY `prefix_index` (`b`(2)) - // } - // - // b(2) > "bbc" -> ["bb", +∞) - // b(2) >= "bbc" -> ["bb", +∞) - // b(2) < "bbc" -> (-∞, "bb"] - // b(2) <= "bbc" -> (-∞, "bb"] - // b(2) = "bbc" -> ["bb", "bb"] - // b(2) > "b" -> ["b", +∞) - // b(2) >= "b" -> ["b", +∞) - // b(2) < "b" -> (-∞, "b"] - // b(2) <= "b" -> (-∞, "b"] - // - // For varchar, `b`(2) will take first two characters(bytes) as prefix index. - // TODO: Note that TiDB only supports UTF-8, we need to check if prefix index behave differently - // under other encoding methods - int prefixLen = lengths.getOrDefault(predicate.getColumnRef(), DataType.UNSPECIFIED_LEN); - TypedKey literal = predicate.getTypedLiteral(prefixLen); - RangeSet ranges = TreeRangeSet.create(); - - if (prefixLen != DataType.UNSPECIFIED_LEN) { - // With prefix length specified, the filter is loosen and so should the ranges - switch (predicate.getType()) { - case GREATER_THAN: - case GREATER_EQUAL: - ranges.add(Range.atLeast(literal)); - break; - case LESS_THAN: - case LESS_EQUAL: - ranges.add(Range.atMost(literal)); - break; - case EQUAL: - ranges.add(Range.singleton(literal)); - break; - case NOT_EQUAL: - // Should return full range because prefix index predicate for NOT_EQUAL - // will be split into an NOT_EQUAL filter and a full range scan - ranges.add(Range.all()); - break; - default: - throwOnError(node); - } - } else { - switch (predicate.getType()) { - case GREATER_THAN: - ranges.add(Range.greaterThan(literal)); - break; - case GREATER_EQUAL: - ranges.add(Range.atLeast(literal)); - break; - case LESS_THAN: - ranges.add(Range.lessThan(literal)); - break; - case LESS_EQUAL: - ranges.add(Range.atMost(literal)); - break; - case EQUAL: - ranges.add(Range.singleton(literal)); - break; - case NOT_EQUAL: - ranges.add(Range.lessThan(literal)); - ranges.add(Range.greaterThan(literal)); - break; - default: - throwOnError(node); - } - } - return ranges; - } - - @Override - protected RangeSet visit(StringRegExpression node, Void context) { - ColumnRef columnRef = node.getColumnRef(); - // In order to match a prefix index, we have to cut the literal by prefix length. - // e.g., for table t: - // CREATE TABLE `t` { - // `c1` VARCHAR(10) DEFAULT NULL, - // KEY `prefix_index` (`c`(2)) - // } - // when the predicate is `c1` LIKE 'abc%', the index range should be ['ab', 'ab']. - // when the predicate is `c1` LIKE 'a%', the index range should be ['a', 'b'). - // for varchar, `c1`(2) will take first two characters(bytes) as prefix index. - // TODO: Note that TiDB only supports UTF-8, we need to check if prefix index behave differently - // under other encoding methods - int prefixLen = lengths.getOrDefault(columnRef, DataType.UNSPECIFIED_LEN); - TypedKey literal = node.getTypedLiteral(prefixLen); - RangeSet ranges = TreeRangeSet.create(); - - switch (node.getRegType()) { - case STARTS_WITH: - ranges.add(Range.atLeast(literal).intersection(Range.lessThan(literal.next(prefixLen)))); - break; - default: - throwOnError(node); - } - return ranges; - } -} diff --git a/src/main/java/org/tikv/expression/visitor/MetaResolver.java b/src/main/java/org/tikv/expression/visitor/MetaResolver.java deleted file mode 100644 index 854d612883..0000000000 --- a/src/main/java/org/tikv/expression/visitor/MetaResolver.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression.visitor; - -import java.util.List; -import java.util.Objects; -import org.tikv.expression.ColumnRef; -import org.tikv.expression.Expression; -import org.tikv.meta.TiTableInfo; - -public class MetaResolver extends DefaultVisitor { - public static void resolve(Expression expression, TiTableInfo table) { - MetaResolver resolver = new MetaResolver(table); - resolver.resolve(expression); - } - - public static void resolve(List expressions, TiTableInfo table) { - MetaResolver resolver = new MetaResolver(table); - resolver.resolve(expressions); - } - - private final TiTableInfo table; - - public MetaResolver(TiTableInfo table) { - this.table = table; - } - - public void resolve(List expressions) { - expressions.forEach(expression -> expression.accept(this, null)); - } - - public void resolve(Expression expression) { - Objects.requireNonNull(expression, "expression is null"); - expression.accept(this, null); - } - - @Override - protected Void visit(ColumnRef node, Expression parent) { - node.resolve(table); - return null; - } -} diff --git a/src/main/java/org/tikv/expression/visitor/ProtoConverter.java b/src/main/java/org/tikv/expression/visitor/ProtoConverter.java deleted file mode 100644 index 8b10906f46..0000000000 --- a/src/main/java/org/tikv/expression/visitor/ProtoConverter.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression.visitor; - -import static java.util.Objects.requireNonNull; - -import com.google.common.collect.ImmutableMap; -import com.pingcap.tidb.tipb.Expr; -import com.pingcap.tidb.tipb.ExprType; -import com.pingcap.tidb.tipb.FieldType; -import com.pingcap.tidb.tipb.ScalarFuncSig; -import java.util.IdentityHashMap; -import java.util.Map; -import org.tikv.codec.Codec.IntegerCodec; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.TiExpressionException; -import org.tikv.expression.*; -import org.tikv.expression.AggregateFunction.FunctionType; -import org.tikv.types.*; -import org.tikv.types.DataType.EncodeType; - -public class ProtoConverter extends Visitor { - // All concrete data type should be hooked to a type name - private static final Map, String> SCALAR_SIG_MAP = - ImmutableMap., String>builder() - .put(IntegerType.class, "Int") - .put(BitType.class, "Int") - .put(DecimalType.class, "Decimal") - .put(RealType.class, "Real") - .put(DateTimeType.class, "Time") - .put(DateType.class, "Time") - .put(TimestampType.class, "Time") - .put(BytesType.class, "String") - .put(StringType.class, "String") - .build(); - - private final IdentityHashMap typeMap; - private final boolean validateColPosition; - - public ProtoConverter(IdentityHashMap typeMap) { - this(typeMap, true); - } - - /** - * Instantiate a {{@code ProtoConverter}} using a typeMap. - * - * @param typeMap the type map - * @param validateColPosition whether to consider column position in this converter. By default, a - * {{@code TiDAGRequest}} should check whether a {{@code ColumnRef}}'s position is correct in - * it's executors. Can ignore this validation if `validateColPosition` is set to false. - */ - public ProtoConverter( - IdentityHashMap typeMap, boolean validateColPosition) { - this.typeMap = typeMap; - this.validateColPosition = validateColPosition; - } - - private DataType getType(Expression expression) { - DataType type = typeMap.get(expression); - if (type == null) { - throw new TiExpressionException(String.format("Expression %s type unknown", expression)); - } - return type; - } - - private String getTypeSignature(Expression expression) { - DataType type = getType(expression); - String typeSignature = SCALAR_SIG_MAP.get(type.getClass()); - if (typeSignature == null) { - throw new TiExpressionException(String.format("Type %s signature unknown", type)); - } - return typeSignature; - } - - public static Expr toProto(Expression expression) { - return toProto(expression, null); - } - - public static Expr toProto(Expression expression, Object context) { - ExpressionTypeCoercer coercer = new ExpressionTypeCoercer(); - coercer.infer(expression); - ProtoConverter converter = new ProtoConverter(coercer.getTypeMap()); - return expression.accept(converter, context); - } - - // Generate protobuf builder with partial data encoded. - // Scala Signature is left alone - private Expr.Builder scalaToPartialProto(Expression node, Object context) { - Expr.Builder builder = Expr.newBuilder(); - // Scalar function type - builder.setTp(ExprType.ScalarFunc); - - // Return type - builder.setFieldType(FieldType.newBuilder().setTp(getType(node).getTypeCode()).build()); - - for (Expression child : node.getChildren()) { - Expr exprProto = child.accept(this, context); - builder.addChildren(exprProto); - } - - return builder; - } - - @Override - protected Expr visit(LogicalBinaryExpression node, Object context) { - ScalarFuncSig protoSig; - switch (node.getCompType()) { - case AND: - protoSig = ScalarFuncSig.LogicalAnd; - break; - case OR: - protoSig = ScalarFuncSig.LogicalOr; - break; - case XOR: - protoSig = ScalarFuncSig.LogicalXor; - break; - default: - throw new TiExpressionException( - String.format("Unknown comparison type %s", node.getCompType())); - } - Expr.Builder builder = scalaToPartialProto(node, context); - builder.setSig(protoSig); - return builder.build(); - } - - @Override - protected Expr visit(ArithmeticBinaryExpression node, Object context) { - // assume after type coerce, children should be compatible - Expression child = node.getLeft(); - String typeSignature = getTypeSignature(child); - ScalarFuncSig protoSig; - switch (node.getCompType()) { - // TODO: Add test for bitwise push down - case BIT_AND: - protoSig = ScalarFuncSig.BitAndSig; - break; - case BIT_OR: - protoSig = ScalarFuncSig.BitOrSig; - break; - case BIT_XOR: - protoSig = ScalarFuncSig.BitXorSig; - break; - case DIVIDE: - protoSig = ScalarFuncSig.valueOf("Divide" + typeSignature); - break; - case MINUS: - protoSig = ScalarFuncSig.valueOf("Minus" + typeSignature); - break; - case MULTIPLY: - protoSig = ScalarFuncSig.valueOf("Multiply" + typeSignature); - break; - case PLUS: - protoSig = ScalarFuncSig.valueOf("Plus" + typeSignature); - break; - default: - throw new TiExpressionException( - String.format("Unknown comparison type %s", node.getCompType())); - } - Expr.Builder builder = scalaToPartialProto(node, context); - builder.setSig(protoSig); - return builder.build(); - } - - @Override - protected Expr visit(ComparisonBinaryExpression node, Object context) { - // assume after type coerce, children should be compatible - Expression child = node.getLeft(); - String typeSignature = getTypeSignature(child); - ScalarFuncSig protoSig; - switch (node.getComparisonType()) { - case EQUAL: - protoSig = ScalarFuncSig.valueOf("EQ" + typeSignature); - break; - case GREATER_EQUAL: - protoSig = ScalarFuncSig.valueOf("GE" + typeSignature); - break; - case GREATER_THAN: - protoSig = ScalarFuncSig.valueOf("GT" + typeSignature); - break; - case LESS_EQUAL: - protoSig = ScalarFuncSig.valueOf("LE" + typeSignature); - break; - case LESS_THAN: - protoSig = ScalarFuncSig.valueOf("LT" + typeSignature); - break; - case NOT_EQUAL: - protoSig = ScalarFuncSig.valueOf("NE" + typeSignature); - break; - default: - throw new TiExpressionException( - String.format("Unknown comparison type %s", node.getComparisonType())); - } - Expr.Builder builder = scalaToPartialProto(node, context); - builder.setSig(protoSig); - return builder.build(); - } - - @Override - protected Expr visit(StringRegExpression node, Object context) { - // assume after type coerce, children should be compatible - ScalarFuncSig protoSig; - switch (node.getRegType()) { - case STARTS_WITH: - case CONTAINS: - case ENDS_WITH: - case LIKE: - protoSig = ScalarFuncSig.LikeSig; - break; - default: - throw new TiExpressionException(String.format("Unknown reg type %s", node.getRegType())); - } - Expr.Builder builder = scalaToPartialProto(node, context); - builder.setSig(protoSig); - return builder.build(); - } - - @Override - @SuppressWarnings("unchecked") - protected Expr visit(ColumnRef node, Object context) { - long position = 0; - if (validateColPosition) { - requireNonNull(context, "Context of a ColumnRef should not be null"); - Map colIdOffsetMap = (Map) context; - position = - requireNonNull( - colIdOffsetMap.get(node), "Required column position info is not in a valid context."); - } - Expr.Builder builder = Expr.newBuilder(); - builder.setTp(ExprType.ColumnRef); - CodecDataOutput cdo = new CodecDataOutput(); - // After switching to DAG request mode, expression value - // should be the index of table columns we provided in - // the first executor of a DAG request. - IntegerCodec.writeLong(cdo, position); - builder.setVal(cdo.toByteString()); - return builder.build(); - } - - @Override - protected Expr visit(Constant node, Object context) { - Expr.Builder builder = Expr.newBuilder(); - if (node.getValue() == null) { - builder.setTp(ExprType.Null); - return builder.build(); - } else { - DataType type = node.getType(); - builder.setTp(type.getProtoExprType()); - CodecDataOutput cdo = new CodecDataOutput(); - type.encode(cdo, EncodeType.PROTO, node.getValue()); - builder.setVal(cdo.toByteString()); - } - return builder.build(); - } - - @Override - protected Expr visit(AggregateFunction node, Object context) { - Expr.Builder builder = Expr.newBuilder(); - - FunctionType type = node.getType(); - switch (type) { - case Max: - builder.setTp(ExprType.Max); - break; - case Sum: - builder.setTp(ExprType.Sum); - break; - case Min: - builder.setTp(ExprType.Min); - break; - case First: - builder.setTp(ExprType.First); - break; - case Count: - builder.setTp(ExprType.Count); - break; - } - - for (Expression arg : node.getChildren()) { - Expr exprProto = arg.accept(this, context); - builder.addChildren(exprProto); - } - - return builder.build(); - } - - @Override - protected Expr visit(IsNull node, Object context) { - String typeSignature = getTypeSignature(node.getExpression()); - ScalarFuncSig protoSig = ScalarFuncSig.valueOf(typeSignature + "IsNull"); - Expr.Builder builder = scalaToPartialProto(node, context); - builder.setSig(protoSig); - return builder.build(); - } - - @Override - protected Expr visit(Not node, Object context) { - ScalarFuncSig protoSig = ScalarFuncSig.UnaryNot; - Expr.Builder builder = scalaToPartialProto(node, context); - builder.setSig(protoSig); - return builder.build(); - } -} diff --git a/src/main/java/org/tikv/expression/visitor/PseudoCostCalculator.java b/src/main/java/org/tikv/expression/visitor/PseudoCostCalculator.java deleted file mode 100644 index 9db9a476e6..0000000000 --- a/src/main/java/org/tikv/expression/visitor/PseudoCostCalculator.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression.visitor; - -import org.tikv.expression.ComparisonBinaryExpression; -import org.tikv.expression.Expression; -import org.tikv.expression.LogicalBinaryExpression; - -public class PseudoCostCalculator extends DefaultVisitor { - public static double calculateCost(Expression expr) { - PseudoCostCalculator calc = new PseudoCostCalculator(); - return expr.accept(calc, null); - } - - @Override - protected Double process(Expression node, Void context) { - return 1.0; - } - - @Override - protected Double visit(LogicalBinaryExpression node, Void context) { - double leftCost = node.getLeft().accept(this, context); - double rightCost = node.getLeft().accept(this, context); - switch (node.getCompType()) { - case AND: - return leftCost * rightCost; - case OR: - case XOR: - return leftCost + rightCost; - default: - return 1.0; - } - } - - @Override - protected Double visit(ComparisonBinaryExpression node, Void context) { - switch (node.getComparisonType()) { - case EQUAL: - return 0.01; - case GREATER_EQUAL: - case GREATER_THAN: - case LESS_EQUAL: - case LESS_THAN: - // magic number for testing - return 0.3; - case NOT_EQUAL: - return 0.99; - default: - return 1.0; - } - } -} diff --git a/src/main/java/org/tikv/expression/visitor/SupportedExpressionValidator.java b/src/main/java/org/tikv/expression/visitor/SupportedExpressionValidator.java deleted file mode 100644 index dcfaa4ddc2..0000000000 --- a/src/main/java/org/tikv/expression/visitor/SupportedExpressionValidator.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression.visitor; - -import org.tikv.expression.Expression; -import org.tikv.expression.ExpressionBlacklist; - -public class SupportedExpressionValidator extends DefaultVisitor { - private static final SupportedExpressionValidator validator = new SupportedExpressionValidator(); - - public static boolean isSupportedExpression(Expression node, ExpressionBlacklist blacklist) { - if (!node.accept(validator, blacklist)) { - return false; - } - try { - ExpressionTypeCoercer coercer = new ExpressionTypeCoercer(); - coercer.infer(node); - ProtoConverter protoConverter = new ProtoConverter(coercer.getTypeMap(), false); - if (node.accept(protoConverter, null) == null) { - return false; - } - } catch (Exception e) { - return false; - } - return true; - } - - @Override - protected Boolean process(Expression node, ExpressionBlacklist blacklist) { - if (blacklist != null && blacklist.isUnsupportedPushdownExpr(getClass())) { - return false; - } - for (Expression expr : node.getChildren()) { - if (!expr.accept(this, blacklist)) { - return false; - } - } - return true; - } -} diff --git a/src/main/java/org/tikv/key/CompoundKey.java b/src/main/java/org/tikv/key/CompoundKey.java deleted file mode 100644 index d55e8429aa..0000000000 --- a/src/main/java/org/tikv/key/CompoundKey.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.key; - -import com.google.common.base.Joiner; -import java.util.ArrayList; -import java.util.List; -import org.tikv.codec.CodecDataOutput; - -public class CompoundKey extends Key { - - private final List keys; - - protected CompoundKey(List keys, byte[] value) { - super(value); - this.keys = keys; - } - - public static CompoundKey concat(Key lKey, Key rKey) { - Builder builder = newBuilder(); - builder.append(lKey).append(rKey); - return builder.build(); - } - - public List getKeys() { - return keys; - } - - public static Builder newBuilder() { - return new Builder(); - } - - public static class Builder { - private final List keys = new ArrayList<>(); - - public Builder append(Key key) { - if (key instanceof CompoundKey) { - CompoundKey compKey = (CompoundKey) key; - for (Key child : compKey.getKeys()) { - append(child); - } - } else { - keys.add(key); - } - return this; - } - - public CompoundKey build() { - int totalLen = 0; - for (Key key : keys) { - totalLen += key.getBytes().length; - } - CodecDataOutput cdo = new CodecDataOutput(totalLen); - for (Key key : keys) { - cdo.write(key.getBytes()); - } - return new CompoundKey(keys, cdo.toBytes()); - } - } - - @Override - public String toString() { - return String.format("[%s]", Joiner.on(",").useForNull("Null").join(keys)); - } -} diff --git a/src/main/java/org/tikv/key/IndexKey.java b/src/main/java/org/tikv/key/IndexKey.java deleted file mode 100644 index 8e98cc7639..0000000000 --- a/src/main/java/org/tikv/key/IndexKey.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.key; - -import com.google.common.base.Joiner; -import org.tikv.codec.Codec.IntegerCodec; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.TypeException; - -public class IndexKey extends Key { - private static final byte[] IDX_PREFIX_SEP = new byte[] {'_', 'i'}; - - private final long tableId; - private final long indexId; - private final Key[] dataKeys; - - private IndexKey(long tableId, long indexId, Key[] dataKeys) { - super(encode(tableId, indexId, dataKeys)); - this.tableId = tableId; - this.indexId = indexId; - this.dataKeys = dataKeys; - } - - public static IndexKey toIndexKey(long tableId, long indexId, Key... dataKeys) { - return new IndexKey(tableId, indexId, dataKeys); - } - - private static byte[] encode(long tableId, long indexId, Key[] dataKeys) { - CodecDataOutput cdo = new CodecDataOutput(); - cdo.write(TBL_PREFIX); - IntegerCodec.writeLong(cdo, tableId); - cdo.write(IDX_PREFIX_SEP); - IntegerCodec.writeLong(cdo, indexId); - for (Key key : dataKeys) { - if (key == null) { - throw new TypeException("key cannot be null"); - } - cdo.write(key.getBytes()); - } - return cdo.toBytes(); - } - - public long getTableId() { - return tableId; - } - - public long getIndexId() { - return indexId; - } - - public Key[] getDataKeys() { - return dataKeys; - } - - @Override - public String toString() { - return String.format("[%s]", Joiner.on(",").useForNull("null").join(dataKeys)); - } -} diff --git a/src/main/java/org/tikv/key/Key.java b/src/main/java/org/tikv/key/Key.java index d540f75a23..678ca42b54 100644 --- a/src/main/java/org/tikv/key/Key.java +++ b/src/main/java/org/tikv/key/Key.java @@ -20,20 +20,14 @@ import static org.tikv.codec.KeyUtils.formatBytes; import com.google.protobuf.ByteString; import java.util.Arrays; -import org.tikv.codec.CodecDataOutput; -import org.tikv.types.DataType; import org.tikv.util.FastByteComparisons; public class Key implements Comparable { - protected static final byte[] TBL_PREFIX = new byte[] {'t'}; protected final byte[] value; protected final int infFlag; public static final Key EMPTY = createEmpty(); - public static final Key NULL = createNull(); - public static final Key MIN = createTypelessMin(); - public static final Key MAX = createTypelessMax(); private Key(byte[] value, boolean negative) { this.value = requireNonNull(value, "value is null"); @@ -60,17 +54,6 @@ public class Key implements Comparable { return new Key(bytes); } - private static Key createNull() { - CodecDataOutput cdo = new CodecDataOutput(); - DataType.encodeNull(cdo); - return new Key(cdo.toBytes()) { - @Override - public String toString() { - return "null"; - } - }; - } - private static Key createEmpty() { return new Key(new byte[0]) { @Override @@ -85,28 +68,6 @@ public class Key implements Comparable { }; } - private static Key createTypelessMin() { - CodecDataOutput cdo = new CodecDataOutput(); - DataType.encodeIndex(cdo); - return new Key(cdo.toBytes()) { - @Override - public String toString() { - return "MIN"; - } - }; - } - - private static Key createTypelessMax() { - CodecDataOutput cdo = new CodecDataOutput(); - DataType.encodeMaxValue(cdo); - return new Key(cdo.toBytes()) { - @Override - public String toString() { - return "MAX"; - } - }; - } - /** * The next key for bytes domain It first plus one at LSB and if LSB overflows, a zero byte is * appended at the end Original bytes will be reused if possible diff --git a/src/main/java/org/tikv/key/RowKey.java b/src/main/java/org/tikv/key/RowKey.java deleted file mode 100644 index 62e56a7635..0000000000 --- a/src/main/java/org/tikv/key/RowKey.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.key; - -import static org.tikv.codec.Codec.IntegerCodec.writeLong; - -import java.util.Objects; -import org.tikv.codec.Codec.IntegerCodec; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.TiClientInternalException; -import org.tikv.exception.TiExpressionException; -import org.tikv.key.RowKey.DecodeResult.Status; -import org.tikv.util.FastByteComparisons; - -public class RowKey extends Key { - private static final byte[] REC_PREFIX_SEP = new byte[] {'_', 'r'}; - - private final long tableId; - private final long handle; - private final boolean maxHandleFlag; - - private RowKey(long tableId, long handle) { - super(encode(tableId, handle)); - this.tableId = tableId; - this.handle = handle; - this.maxHandleFlag = false; - } - - /** - * The RowKey indicating maximum handle (its value exceeds Long.Max_Value) - * - *

Initializes an imaginary globally MAXIMUM rowKey with tableId. - */ - private RowKey(long tableId) { - super(encodeBeyondMaxHandle(tableId)); - this.tableId = tableId; - this.handle = Long.MAX_VALUE; - this.maxHandleFlag = true; - } - - public static RowKey toRowKey(long tableId, long handle) { - return new RowKey(tableId, handle); - } - - public static RowKey toRowKey(long tableId, TypedKey handle) { - Object obj = handle.getValue(); - if (obj instanceof Long) { - return new RowKey(tableId, (long) obj); - } - throw new TiExpressionException("Cannot encode row key with non-long type"); - } - - public static RowKey createMin(long tableId) { - return toRowKey(tableId, Long.MIN_VALUE); - } - - public static RowKey createBeyondMax(long tableId) { - return new RowKey(tableId); - } - - private static byte[] encode(long tableId, long handle) { - CodecDataOutput cdo = new CodecDataOutput(); - encodePrefix(cdo, tableId); - writeLong(cdo, handle); - return cdo.toBytes(); - } - - private static byte[] encodeBeyondMaxHandle(long tableId) { - return nextValue(encode(tableId, Long.MAX_VALUE)); - } - - @Override - public RowKey next() { - long handle = getHandle(); - boolean maxHandleFlag = getMaxHandleFlag(); - if (maxHandleFlag) { - throw new TiClientInternalException("Handle overflow for Long MAX"); - } - if (handle == Long.MAX_VALUE) { - return createBeyondMax(tableId); - } - return new RowKey(tableId, handle + 1); - } - - public long getTableId() { - return tableId; - } - - public long getHandle() { - return handle; - } - - private boolean getMaxHandleFlag() { - return maxHandleFlag; - } - - @Override - public String toString() { - return Long.toString(handle); - } - - private static void encodePrefix(CodecDataOutput cdo, long tableId) { - cdo.write(TBL_PREFIX); - writeLong(cdo, tableId); - cdo.write(REC_PREFIX_SEP); - } - - public static class DecodeResult { - public long handle; - - public enum Status { - MIN, - MAX, - EQUAL, - LESS, - GREATER, - UNKNOWN_INF - } - - public Status status; - } - - public static void tryDecodeRowKey(long tableId, byte[] rowKey, DecodeResult outResult) { - Objects.requireNonNull(rowKey, "rowKey cannot be null"); - if (rowKey.length == 0) { - outResult.status = Status.UNKNOWN_INF; - return; - } - CodecDataOutput cdo = new CodecDataOutput(); - encodePrefix(cdo, tableId); - byte[] tablePrefix = cdo.toBytes(); - - int res = - FastByteComparisons.compareTo( - tablePrefix, - 0, - tablePrefix.length, - rowKey, - 0, - Math.min(rowKey.length, tablePrefix.length)); - - if (res > 0) { - outResult.status = Status.MIN; - return; - } - if (res < 0) { - outResult.status = Status.MAX; - return; - } - - CodecDataInput cdi = new CodecDataInput(rowKey); - cdi.skipBytes(tablePrefix.length); - if (cdi.available() == 8) { - outResult.status = Status.EQUAL; - } else if (cdi.available() < 8) { - outResult.status = Status.LESS; - } else if (cdi.available() > 8) { - outResult.status = Status.GREATER; - } - outResult.handle = IntegerCodec.readPartialLong(cdi); - } -} diff --git a/src/main/java/org/tikv/key/TypedKey.java b/src/main/java/org/tikv/key/TypedKey.java deleted file mode 100644 index 88a0dca444..0000000000 --- a/src/main/java/org/tikv/key/TypedKey.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.key; - -import static java.util.Objects.requireNonNull; - -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.TypeException; -import org.tikv.types.DataType; - -public class TypedKey extends Key { - private final DataType type; - - public TypedKey(Object val, DataType type, int prefixLength) { - super(encodeKey(val, type, prefixLength)); - this.type = type; - } - - public DataType getType() { - return type; - } - - public Object getValue() { - CodecDataInput cdi = new CodecDataInput(value); - return type.decode(cdi); - } - - public static TypedKey toTypedKey(Object val, DataType type) { - return toTypedKey(val, type, DataType.UNSPECIFIED_LEN); - } - - /** - * Map a typed value into TypedKey, only encoding first prefixLength bytes When prefixLength is - * DataType.UNSPECIFIED_LEN, encode full length of value - * - * @param val value - * @param type type of value - * @param prefixLength described above - * @return an encoded TypedKey - */ - public static TypedKey toTypedKey(Object val, DataType type, int prefixLength) { - requireNonNull(type, "type is null"); - return new TypedKey(val, type, prefixLength); - } - - private static byte[] encodeKey(Object val, DataType type, int prefixLength) { - CodecDataOutput cdo = new CodecDataOutput(); - type.encodeKey(cdo, val, type, prefixLength); - return cdo.toBytes(); - } - - public TypedKey next(int prefixLength) { - Object val = getValue(); - if (val instanceof String) { - return toTypedKey(nextValue(((String) val).getBytes()), type, prefixLength); - } else if (val instanceof byte[]) { - return toTypedKey(nextValue(((byte[]) val)), type, prefixLength); - } else { - throw new TypeException( - "Type for TypedKey in next() function must be either String or Byte array"); - } - } - - @Override - public String toString() { - CodecDataInput cdi = new CodecDataInput(value); - Object val = type.decode(cdi); - return String.format("%s", val); - } -} diff --git a/src/main/java/org/tikv/meta/CIStr.java b/src/main/java/org/tikv/meta/CIStr.java deleted file mode 100644 index ae7267dd7e..0000000000 --- a/src/main/java/org/tikv/meta/CIStr.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; - -/** This class is mapping TiDB's CIStr/ For internal use only. */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class CIStr { - private final String o; // original - private final String l; - - @JsonCreator - private CIStr(@JsonProperty("O") String o, @JsonProperty("L") String l) { - this.o = o; - this.l = l; - } - - public static CIStr newCIStr(String str) { - return new CIStr(str, str.toLowerCase()); - } - - public String getO() { - return o; - } - - public String getL() { - return l; - } -} diff --git a/src/main/java/org/tikv/meta/Collation.java b/src/main/java/org/tikv/meta/Collation.java deleted file mode 100644 index aff066f35c..0000000000 --- a/src/main/java/org/tikv/meta/Collation.java +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import com.google.common.collect.ImmutableMap; -import java.util.Map; - -public class Collation { - public static int translate(String collation) { - Integer code = collationMap.get(collation); - if (code == null) { - return DEF_COLLATION_CODE; - } - return code; - } - - public static String translate(int code) { - String collation = collationCodeMap.get(code); - if (collation == null) { - return ""; - } - return collation; - } - - public static final int DEF_COLLATION_CODE = 83; - - private static final Map collationMap; - private static final Map collationCodeMap; - - static { - collationMap = - ImmutableMap.builder() - .put("big5_chinese_ci", 1) - .put("latin2_czech_cs", 2) - .put("dec8_swedish_ci", 3) - .put("cp850_general_ci", 4) - .put("latin1_german1_ci", 5) - .put("hp8_english_ci", 6) - .put("koi8r_general_ci", 7) - .put("latin1_swedish_ci", 8) - .put("latin2_general_ci", 9) - .put("swe7_swedish_ci", 10) - .put("ascii_general_ci", 11) - .put("ujis_japanese_ci", 12) - .put("sjis_japanese_ci", 13) - .put("cp1251_bulgarian_ci", 14) - .put("latin1_danish_ci", 15) - .put("hebrew_general_ci", 16) - .put("tis620_thai_ci", 18) - .put("euckr_korean_ci", 19) - .put("latin7_estonian_cs", 20) - .put("latin2_hungarian_ci", 21) - .put("koi8u_general_ci", 22) - .put("cp1251_ukrainian_ci", 23) - .put("gb2312_chinese_ci", 24) - .put("greek_general_ci", 25) - .put("cp1250_general_ci", 26) - .put("latin2_croatian_ci", 27) - .put("gbk_chinese_ci", 28) - .put("cp1257_lithuanian_ci", 29) - .put("latin5_turkish_ci", 30) - .put("latin1_german2_ci", 31) - .put("armscii8_general_ci", 32) - .put("utf8_general_ci", 33) - .put("cp1250_czech_cs", 34) - .put("ucs2_general_ci", 35) - .put("cp866_general_ci", 36) - .put("keybcs2_general_ci", 37) - .put("macce_general_ci", 38) - .put("macroman_general_ci", 39) - .put("cp852_general_ci", 40) - .put("latin7_general_ci", 41) - .put("latin7_general_cs", 42) - .put("macce_bin", 43) - .put("cp1250_croatian_ci", 44) - .put("utf8mb4_general_ci", 45) - .put("utf8mb4_bin", 46) - .put("latin1_bin", 47) - .put("latin1_general_ci", 48) - .put("latin1_general_cs", 49) - .put("cp1251_bin", 50) - .put("cp1251_general_ci", 51) - .put("cp1251_general_cs", 52) - .put("macroman_bin", 53) - .put("utf16_general_ci", 54) - .put("utf16_bin", 55) - .put("utf16le_general_ci", 56) - .put("cp1256_general_ci", 57) - .put("cp1257_bin", 58) - .put("cp1257_general_ci", 59) - .put("utf32_general_ci", 60) - .put("utf32_bin", 61) - .put("utf16le_bin", 62) - .put("binary", 63) - .put("armscii8_bin", 64) - .put("ascii_bin", 65) - .put("cp1250_bin", 66) - .put("cp1256_bin", 67) - .put("cp866_bin", 68) - .put("dec8_bin", 69) - .put("greek_bin", 70) - .put("hebrew_bin", 71) - .put("hp8_bin", 72) - .put("keybcs2_bin", 73) - .put("koi8r_bin", 74) - .put("koi8u_bin", 75) - .put("latin2_bin", 77) - .put("latin5_bin", 78) - .put("latin7_bin", 79) - .put("cp850_bin", 80) - .put("cp852_bin", 81) - .put("swe7_bin", 82) - .put("utf8_bin", 83) - .put("big5_bin", 84) - .put("euckr_bin", 85) - .put("gb2312_bin", 86) - .put("gbk_bin", 87) - .put("sjis_bin", 88) - .put("tis620_bin", 89) - .put("ucs2_bin", 90) - .put("ujis_bin", 91) - .put("geostd8_general_ci", 92) - .put("geostd8_bin", 93) - .put("latin1_spanish_ci", 94) - .put("cp932_japanese_ci", 95) - .put("cp932_bin", 96) - .put("eucjpms_japanese_ci", 97) - .put("eucjpms_bin", 98) - .put("cp1250_polish_ci", 99) - .put("utf16_unicode_ci", 101) - .put("utf16_icelandic_ci", 102) - .put("utf16_latvian_ci", 103) - .put("utf16_romanian_ci", 104) - .put("utf16_slovenian_ci", 105) - .put("utf16_polish_ci", 106) - .put("utf16_estonian_ci", 107) - .put("utf16_spanish_ci", 108) - .put("utf16_swedish_ci", 109) - .put("utf16_turkish_ci", 110) - .put("utf16_czech_ci", 111) - .put("utf16_danish_ci", 112) - .put("utf16_lithuanian_ci", 113) - .put("utf16_slovak_ci", 114) - .put("utf16_spanish2_ci", 115) - .put("utf16_roman_ci", 116) - .put("utf16_persian_ci", 117) - .put("utf16_esperanto_ci", 118) - .put("utf16_hungarian_ci", 119) - .put("utf16_sinhala_ci", 120) - .put("utf16_german2_ci", 121) - .put("utf16_croatian_ci", 122) - .put("utf16_unicode_520_ci", 123) - .put("utf16_vietnamese_ci", 124) - .put("ucs2_unicode_ci", 128) - .put("ucs2_icelandic_ci", 129) - .put("ucs2_latvian_ci", 130) - .put("ucs2_romanian_ci", 131) - .put("ucs2_slovenian_ci", 132) - .put("ucs2_polish_ci", 133) - .put("ucs2_estonian_ci", 134) - .put("ucs2_spanish_ci", 135) - .put("ucs2_swedish_ci", 136) - .put("ucs2_turkish_ci", 137) - .put("ucs2_czech_ci", 138) - .put("ucs2_danish_ci", 139) - .put("ucs2_lithuanian_ci", 140) - .put("ucs2_slovak_ci", 141) - .put("ucs2_spanish2_ci", 142) - .put("ucs2_roman_ci", 143) - .put("ucs2_persian_ci", 144) - .put("ucs2_esperanto_ci", 145) - .put("ucs2_hungarian_ci", 146) - .put("ucs2_sinhala_ci", 147) - .put("ucs2_german2_ci", 148) - .put("ucs2_croatian_ci", 149) - .put("ucs2_unicode_520_ci", 150) - .put("ucs2_vietnamese_ci", 151) - .put("ucs2_general_mysql500_ci", 159) - .put("utf32_unicode_ci", 160) - .put("utf32_icelandic_ci", 161) - .put("utf32_latvian_ci", 162) - .put("utf32_romanian_ci", 163) - .put("utf32_slovenian_ci", 164) - .put("utf32_polish_ci", 165) - .put("utf32_estonian_ci", 166) - .put("utf32_spanish_ci", 167) - .put("utf32_swedish_ci", 168) - .put("utf32_turkish_ci", 169) - .put("utf32_czech_ci", 170) - .put("utf32_danish_ci", 171) - .put("utf32_lithuanian_ci", 172) - .put("utf32_slovak_ci", 173) - .put("utf32_spanish2_ci", 174) - .put("utf32_roman_ci", 175) - .put("utf32_persian_ci", 176) - .put("utf32_esperanto_ci", 177) - .put("utf32_hungarian_ci", 178) - .put("utf32_sinhala_ci", 179) - .put("utf32_german2_ci", 180) - .put("utf32_croatian_ci", 181) - .put("utf32_unicode_520_ci", 182) - .put("utf32_vietnamese_ci", 183) - .put("utf8_unicode_ci", 192) - .put("utf8_icelandic_ci", 193) - .put("utf8_latvian_ci", 194) - .put("utf8_romanian_ci", 195) - .put("utf8_slovenian_ci", 196) - .put("utf8_polish_ci", 197) - .put("utf8_estonian_ci", 198) - .put("utf8_spanish_ci", 199) - .put("utf8_swedish_ci", 200) - .put("utf8_turkish_ci", 201) - .put("utf8_czech_ci", 202) - .put("utf8_danish_ci", 203) - .put("utf8_lithuanian_ci", 204) - .put("utf8_slovak_ci", 205) - .put("utf8_spanish2_ci", 206) - .put("utf8_roman_ci", 207) - .put("utf8_persian_ci", 208) - .put("utf8_esperanto_ci", 209) - .put("utf8_hungarian_ci", 210) - .put("utf8_sinhala_ci", 211) - .put("utf8_german2_ci", 212) - .put("utf8_croatian_ci", 213) - .put("utf8_unicode_520_ci", 214) - .put("utf8_vietnamese_ci", 215) - .put("utf8_general_mysql500_ci", 223) - .put("utf8mb4_unicode_ci", 224) - .put("utf8mb4_icelandic_ci", 225) - .put("utf8mb4_latvian_ci", 226) - .put("utf8mb4_romanian_ci", 227) - .put("utf8mb4_slovenian_ci", 228) - .put("utf8mb4_polish_ci", 229) - .put("utf8mb4_estonian_ci", 230) - .put("utf8mb4_spanish_ci", 231) - .put("utf8mb4_swedish_ci", 232) - .put("utf8mb4_turkish_ci", 233) - .put("utf8mb4_czech_ci", 234) - .put("utf8mb4_danish_ci", 235) - .put("utf8mb4_lithuanian_ci", 236) - .put("utf8mb4_slovak_ci", 237) - .put("utf8mb4_spanish2_ci", 238) - .put("utf8mb4_roman_ci", 239) - .put("utf8mb4_persian_ci", 240) - .put("utf8mb4_esperanto_ci", 241) - .put("utf8mb4_hungarian_ci", 242) - .put("utf8mb4_sinhala_ci", 243) - .put("utf8mb4_german2_ci", 244) - .put("utf8mb4_croatian_ci", 245) - .put("utf8mb4_unicode_520_ci", 246) - .put("utf8mb4_vietnamese_ci", 247) - .build(); - - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (String collation : collationMap.keySet()) { - builder.put(collationMap.get(collation), collation); - } - collationCodeMap = builder.build(); - } -} diff --git a/src/main/java/org/tikv/meta/IndexType.java b/src/main/java/org/tikv/meta/IndexType.java deleted file mode 100644 index e7353f20ab..0000000000 --- a/src/main/java/org/tikv/meta/IndexType.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import org.tikv.exception.TiClientInternalException; - -// Actually we are not using either real btree or hash index -// TiDB has its own way for indexing as key value pair. -public enum IndexType { - IndexTypeInvalid(0), - IndexTypeBtree(1), - IndexTypeHash(2); - - private final int type; - - IndexType(int type) { - this.type = type; - } - - public static IndexType fromValue(int type) { - for (IndexType e : IndexType.values()) { - if (e.type == type) { - return e; - } - } - throw new TiClientInternalException("Invalid index type code: " + type); - } - - public int getTypeCode() { - return type; - } - - public String toString() { - switch (this.type) { - case 1: - return "BTREE"; - case 2: - return "HASH"; - } - return "Invalid"; - } -} diff --git a/src/main/java/org/tikv/meta/SchemaState.java b/src/main/java/org/tikv/meta/SchemaState.java deleted file mode 100644 index 37448d5c9e..0000000000 --- a/src/main/java/org/tikv/meta/SchemaState.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import org.tikv.exception.TiClientInternalException; - -public enum SchemaState { - StateNone(0), - StateDeleteOnly(1), - StateWriteOnly(2), - StateWriteReorganization(3), - StateDeleteReorganization(4), - StatePublic(5); - - private final int state; - - SchemaState(int state) { - this.state = state; - } - - public static SchemaState fromValue(int b) { - for (SchemaState e : SchemaState.values()) { - if (e.state == b) { - return e; - } - } - throw new TiClientInternalException("Invalid SchemaState code: " + b); - } - - public int getStateCode() { - return state; - } -} diff --git a/src/main/java/org/tikv/meta/TiColumnInfo.java b/src/main/java/org/tikv/meta/TiColumnInfo.java deleted file mode 100644 index b28e65c24a..0000000000 --- a/src/main/java/org/tikv/meta/TiColumnInfo.java +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import static java.util.Objects.requireNonNull; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.ByteString; -import com.pingcap.tidb.tipb.ColumnInfo; -import java.io.Serializable; -import java.util.List; -import java.util.Objects; -import org.tikv.codec.CodecDataOutput; -import org.tikv.types.DataType; -import org.tikv.types.DataType.EncodeType; -import org.tikv.types.DataTypeFactory; -import org.tikv.types.IntegerType; - -@JsonIgnoreProperties(ignoreUnknown = true) -public class TiColumnInfo implements Serializable { - private final long id; - private final String name; - private final int offset; - private final DataType type; - private final SchemaState schemaState; - private final String comment; - private final boolean isPrimaryKey; - private final String defaultValue; - private final String originDefaultValue; - - public static TiColumnInfo getRowIdColumn(int offset) { - return new TiColumnInfo(-1, "_tidb_rowid", offset, IntegerType.ROW_ID_TYPE, true); - } - - @VisibleForTesting private static final int PK_MASK = 0x2; - - @JsonCreator - public TiColumnInfo( - @JsonProperty("id") long id, - @JsonProperty("name") CIStr name, - @JsonProperty("offset") int offset, - @JsonProperty("type") InternalTypeHolder type, - @JsonProperty("state") int schemaState, - @JsonProperty("origin_default") String originalDefaultValue, - @JsonProperty("default") String defaultValue, - @JsonProperty("comment") String comment) { - this.id = id; - this.name = requireNonNull(name, "column name is null").getL(); - this.offset = offset; - this.type = DataTypeFactory.of(requireNonNull(type, "type is null")); - this.schemaState = SchemaState.fromValue(schemaState); - this.comment = comment; - this.defaultValue = defaultValue; - this.originDefaultValue = originalDefaultValue; - // I don't think pk flag should be set on type - // Refactor against original tidb code - this.isPrimaryKey = (type.getFlag() & PK_MASK) > 0; - } - - public TiColumnInfo( - long id, - String name, - int offset, - DataType type, - SchemaState schemaState, - String originalDefaultValue, - String defaultValue, - String comment) { - this.id = id; - this.name = requireNonNull(name, "column name is null").toLowerCase(); - this.offset = offset; - this.type = requireNonNull(type, "data type is null"); - this.schemaState = schemaState; - this.comment = comment; - this.defaultValue = defaultValue; - this.originDefaultValue = originalDefaultValue; - this.isPrimaryKey = (type.getFlag() & PK_MASK) > 0; - } - - public TiColumnInfo copyWithoutPrimaryKey() { - InternalTypeHolder typeHolder = type.toTypeHolder(); - typeHolder.setFlag(type.getFlag() & (~TiColumnInfo.PK_MASK)); - DataType newType = DataTypeFactory.of(typeHolder); - return new TiColumnInfo( - this.id, - this.name, - this.offset, - newType, - this.schemaState, - this.originDefaultValue, - this.defaultValue, - this.comment); - } - - @VisibleForTesting - public TiColumnInfo(long id, String name, int offset, DataType type, boolean isPrimaryKey) { - this.id = id; - this.name = requireNonNull(name, "column name is null").toLowerCase(); - this.offset = offset; - this.type = requireNonNull(type, "data type is null"); - this.schemaState = SchemaState.StatePublic; - this.comment = ""; - this.isPrimaryKey = isPrimaryKey; - this.originDefaultValue = "1"; - this.defaultValue = ""; - } - - public long getId() { - return this.id; - } - - public String getName() { - return this.name; - } - - public boolean matchName(String name) { - return this.name.equalsIgnoreCase(name); - } - - public int getOffset() { - return this.offset; - } - - public DataType getType() { - return type; - } - - public SchemaState getSchemaState() { - return schemaState; - } - - public String getComment() { - return comment; - } - - public boolean isPrimaryKey() { - return isPrimaryKey; - } - - public String getDefaultValue() { - return defaultValue; - } - - public String getOriginDefaultValue() { - return originDefaultValue; - } - - public ByteString getOriginDefaultValueAsByteString() { - CodecDataOutput cdo = new CodecDataOutput(); - type.encode(cdo, EncodeType.VALUE, type.getOriginDefaultValue(originDefaultValue)); - return cdo.toByteString(); - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class InternalTypeHolder { - private int tp; - private int flag; - private long flen; - private int decimal; - private String charset; - private String collate; - private String defaultValue; - private String originDefaultValue; - private List elems; - - public void setTp(int tp) { - this.tp = tp; - } - - public void setFlag(int flag) { - this.flag = flag; - } - - public void setFlen(long flen) { - this.flen = flen; - } - - public void setDecimal(int decimal) { - this.decimal = decimal; - } - - public void setCharset(String charset) { - this.charset = charset; - } - - public void setCollate(String collate) { - this.collate = collate; - } - - public void setDefaultValue(String defaultValue) { - this.defaultValue = defaultValue; - } - - public void setOriginDefaultValue(String originDefaultValue) { - this.originDefaultValue = originDefaultValue; - } - - public void setElems(List elems) { - this.elems = elems; - } - - interface Builder { - E build(InternalTypeHolder holder); - } - - @JsonCreator - public InternalTypeHolder( - @JsonProperty("Tp") int tp, - @JsonProperty("Flag") int flag, - @JsonProperty("Flen") long flen, - @JsonProperty("Decimal") int decimal, - @JsonProperty("Charset") String charset, - @JsonProperty("origin_default") String originalDefaultValue, - @JsonProperty("default") String defaultValue, - @JsonProperty("Collate") String collate, - @JsonProperty("Elems") List elems) { - this.tp = tp; - this.flag = flag; - this.flen = flen; - this.decimal = decimal; - this.charset = charset; - this.collate = collate; - this.defaultValue = defaultValue; - this.originDefaultValue = originalDefaultValue; - this.elems = elems; - } - - public InternalTypeHolder(ColumnInfo c) { - this.tp = c.getTp(); - this.flag = c.getFlag(); - this.flen = c.getColumnLen(); - this.decimal = c.getDecimal(); - this.charset = ""; - this.collate = Collation.translate(c.getCollation()); - this.elems = c.getElemsList(); - this.defaultValue = c.getDefaultVal().toStringUtf8(); - // TODO: we may need write a functon about get origin default value according to the string. - this.originDefaultValue = ""; - } - - public int getTp() { - return tp; - } - - public int getFlag() { - return flag; - } - - public long getFlen() { - return flen; - } - - public int getDecimal() { - return decimal; - } - - public String getCharset() { - return charset; - } - - public String getCollate() { - return collate; - } - - public List getElems() { - return elems; - } - - public String getDefaultValue() { - return defaultValue; - } - - public String getOriginDefaultValue() { - return originDefaultValue; - } - } - - TiIndexColumn toFakeIndexColumn() { - // we don't use original length of column since for a clustered index column - // it always full index instead of prefix index - return new TiIndexColumn(CIStr.newCIStr(getName()), getOffset(), DataType.UNSPECIFIED_LEN); - } - - TiIndexColumn toIndexColumn() { - return new TiIndexColumn(CIStr.newCIStr(getName()), getOffset(), getType().getLength()); - } - - public ColumnInfo toProto(TiTableInfo table) { - return toProtoBuilder(table).build(); - } - - ColumnInfo.Builder toProtoBuilder(TiTableInfo table) { - return ColumnInfo.newBuilder() - .setColumnId(id) - .setTp(type.getTypeCode()) - .setCollation(type.getCollationCode()) - .setColumnLen((int) type.getLength()) - .setDecimal(type.getDecimal()) - .setFlag(type.getFlag()) - .setDefaultVal(getOriginDefaultValueAsByteString()) - .setPkHandle(table.isPkHandle() && isPrimaryKey()) - .addAllElems(type.getElems()); - } - - @Override - public boolean equals(Object other) { - if (other == this) { - return true; - } - - if (!(other instanceof TiColumnInfo)) { - return false; - } - - TiColumnInfo col = (TiColumnInfo) other; - return Objects.equals(id, col.id) - && Objects.equals(name, col.name) - && Objects.equals(type, col.type) - && Objects.equals(schemaState, col.schemaState) - && isPrimaryKey == col.isPrimaryKey - && Objects.equals(defaultValue, col.defaultValue) - && Objects.equals(originDefaultValue, col.originDefaultValue); - } - - @Override - public int hashCode() { - return Objects.hash( - id, name, type, schemaState, isPrimaryKey, defaultValue, originDefaultValue); - } -} diff --git a/src/main/java/org/tikv/meta/TiDAGRequest.java b/src/main/java/org/tikv/meta/TiDAGRequest.java deleted file mode 100644 index a793ad8230..0000000000 --- a/src/main/java/org/tikv/meta/TiDAGRequest.java +++ /dev/null @@ -1,822 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; -import static org.tikv.predicates.PredicateUtils.mergeCNFExpressions; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.pingcap.tidb.tipb.*; -import java.io.*; -import java.util.*; -import java.util.stream.Collectors; -import org.tikv.codec.KeyUtils; -import org.tikv.exception.DAGRequestException; -import org.tikv.exception.TiClientInternalException; -import org.tikv.expression.ByItem; -import org.tikv.expression.ColumnRef; -import org.tikv.expression.Expression; -import org.tikv.expression.visitor.ExpressionTypeCoercer; -import org.tikv.expression.visitor.MetaResolver; -import org.tikv.expression.visitor.ProtoConverter; -import org.tikv.key.RowKey; -import org.tikv.kvproto.Coprocessor; -import org.tikv.types.DataType; -import org.tikv.util.KeyRangeUtils; -import org.tikv.util.Pair; - -/** - * Type TiDAGRequest. - * - *

Used for constructing a new DAG request to TiKV - */ -public class TiDAGRequest implements Serializable { - public static class Builder { - private List requiredCols = new ArrayList<>(); - private List filters = new ArrayList<>(); - private List orderBys = new ArrayList<>(); - private List ranges = new ArrayList<>(); - private TiTableInfo tableInfo; - private int limit; - private long startTs; - - public static Builder newBuilder() { - return new Builder(); - } - - public Builder setFullTableScan(TiTableInfo tableInfo) { - requireNonNull(tableInfo); - setTableInfo(tableInfo); - RowKey start = RowKey.createMin(tableInfo.getId()); - RowKey end = RowKey.createBeyondMax(tableInfo.getId()); - ranges.add(KeyRangeUtils.makeCoprocRange(start.toByteString(), end.toByteString())); - return this; - } - - public Builder setLimit(int limit) { - this.limit = limit; - return this; - } - - public Builder setTableInfo(TiTableInfo tableInfo) { - this.tableInfo = tableInfo; - return this; - } - - public Builder addRequiredCols(String... cols) { - this.requiredCols.addAll(Arrays.asList(cols)); - return this; - } - - public Builder addRequiredCols(List cols) { - this.requiredCols.addAll(cols); - return this; - } - - public Builder addFilter(Expression filter) { - this.filters.add(filter); - return this; - } - - public Builder addOrderBy(ByItem item) { - this.orderBys.add(item); - return this; - } - - public Builder setStartTs(long ts) { - this.startTs = ts; - return this; - } - - public TiDAGRequest build(PushDownType pushDownType) { - TiDAGRequest req = new TiDAGRequest(pushDownType); - req.setTableInfo(tableInfo); - req.addRanges(ranges); - filters.forEach(req::addFilter); - if (!orderBys.isEmpty()) { - orderBys.forEach(req::addOrderByItem); - } - if (limit != 0) { - req.setLimit(limit); - } - requiredCols.forEach(c -> req.addRequiredColumn(ColumnRef.create(c))); - req.setStartTs(startTs); - - req.resolve(); - return req; - } - } - - public TiDAGRequest(PushDownType pushDownType) { - this.pushDownType = pushDownType; - } - - public TiDAGRequest(PushDownType pushDownType, int timeZoneOffset) { - this(pushDownType); - this.timeZoneOffset = timeZoneOffset; - } - - public enum TruncateMode { - IgnoreTruncation(0x1), - TruncationAsWarning(0x2); - - private final long mask; - - TruncateMode(long mask) { - this.mask = mask; - } - - public long mask(long flags) { - return flags | mask; - } - } - - /** Whether we use streaming to push down the request */ - public enum PushDownType { - STREAMING, - NORMAL - } - - /** Predefined executor priority map. */ - private static final Map EXEC_TYPE_PRIORITY_MAP = - ImmutableMap.builder() - .put(ExecType.TypeTableScan, 0) - .put(ExecType.TypeIndexScan, 0) - .put(ExecType.TypeSelection, 1) - .put(ExecType.TypeAggregation, 2) - .put(ExecType.TypeTopN, 3) - .put(ExecType.TypeLimit, 4) - .build(); - - private TiTableInfo tableInfo; - private TiIndexInfo indexInfo; - private final List fields = new ArrayList<>(); - private final List filters = new ArrayList<>(); - private final List groupByItems = new ArrayList<>(); - private final List orderByItems = new ArrayList<>(); - private List pushdownFilters = null; - // System like Spark has different type promotion rules - // we need a cast to target when given - private final List> aggregates = new ArrayList<>(); - private final List keyRanges = new ArrayList<>(); - // If index scanning of this request is not possible in some scenario, we downgrade it to a table - // scan and use - // downGradeRanges instead of index scan ranges stored in keyRanges along with downgradeFilters to - // perform a - // table scan. - private List downgradeFilters = new ArrayList<>(); - - private int limit; - private int timeZoneOffset; - private long flags; - private long startTs; - private Expression having; - private boolean distinct; - private boolean handleNeeded; - private boolean isDoubleRead; - private final PushDownType pushDownType; - private IdentityHashMap typeMap; - private double estimatedCount = -1; - - private static ColumnInfo handleColumn = - ColumnInfo.newBuilder() - .setColumnId(-1) - .setPkHandle(true) - // We haven't changed the field name in protobuf file, but - // we need to set this to true in order to retrieve the handle, - // so the name 'setPkHandle' may sounds strange. - .build(); - - private List getAllExpressions() { - ImmutableList.Builder builder = ImmutableList.builder(); - builder.addAll(getFields()); - builder.addAll(getFilters()); - builder.addAll(getAggregates()); - getGroupByItems().forEach(item -> builder.add(item.getExpr())); - getOrderByItems().forEach(item -> builder.add(item.getExpr())); - if (having != null) { - builder.add(having); - } - return builder.build(); - } - - public DataType getExpressionType(Expression expression) { - requireNonNull(typeMap, "request is not resolved"); - return typeMap.get(expression); - } - - public void resolve() { - MetaResolver resolver = new MetaResolver(tableInfo); - ExpressionTypeCoercer inferrer = new ExpressionTypeCoercer(); - resolver.resolve(getAllExpressions()); - inferrer.infer(getAllExpressions()); - typeMap = inferrer.getTypeMap(); - } - - /** - * Unify indexScan and tableScan building logic since they are very much alike. DAGRequest for - * IndexScan should also contain filters and aggregation, so we can reuse this part of logic. - * - *

DAGRequest is made up of a chain of executors with strict orders: TableScan/IndexScan > - * Selection > Aggregation > TopN/Limit a DAGRequest must contain one and only one TableScan or - * IndexScan. - * - * @param isIndexScan whether the dagRequest to build is an IndexScan - * @return final DAGRequest built - */ - public DAGRequest buildScan(boolean isIndexScan) { - checkArgument(startTs != 0, "timestamp is 0"); - DAGRequest.Builder dagRequestBuilder = DAGRequest.newBuilder(); - Executor.Builder executorBuilder = Executor.newBuilder(); - IndexScan.Builder indexScanBuilder = IndexScan.newBuilder(); - TableScan.Builder tblScanBuilder = TableScan.newBuilder(); - // find a column's offset in fields - Map colOffsetInFieldMap = new HashMap<>(); - // find a column's position in index - Map colPosInIndexMap = new HashMap<>(); - - if (isIndexScan) { - // IndexScan - if (indexInfo == null) { - throw new TiClientInternalException("Index is empty for index scan"); - } - List columnInfoList = tableInfo.getColumns(); - boolean hasPk = false; - // We extract index column info - List indexColOffsets = - indexInfo - .getIndexColumns() - .stream() - .map(TiIndexColumn::getOffset) - .collect(Collectors.toList()); - - int idxPos = 0; - // for index scan builder, columns are added by its order in index - for (Integer idx : indexColOffsets) { - TiColumnInfo tiColumnInfo = columnInfoList.get(idx); - ColumnInfo columnInfo = tiColumnInfo.toProto(tableInfo); - colPosInIndexMap.put(tiColumnInfo, idxPos++); - - ColumnInfo.Builder colBuilder = ColumnInfo.newBuilder(columnInfo); - if (columnInfo.getColumnId() == -1) { - hasPk = true; - colBuilder.setPkHandle(true); - } - indexScanBuilder.addColumns(colBuilder); - } - - if (isDoubleRead()) { - // double read case - if (!hasPk) { - indexScanBuilder.addColumns(handleColumn); - } - - int colCount = indexScanBuilder.getColumnsCount(); - // double read case: need to retrieve handle - dagRequestBuilder.addOutputOffsets(colCount != 0 ? colCount - 1 : 0); - } else { - int colCount = indexScanBuilder.getColumnsCount(); - boolean pkIsNeeded = false; - // =================== IMPORTANT ====================== - // offset for dagRequest should be in accordance with fields - for (ColumnRef col : getFields()) { - Integer pos = colPosInIndexMap.get(col.getColumnInfo()); - if (pos != null) { - TiColumnInfo columnInfo = columnInfoList.get(indexColOffsets.get(pos)); - if (col.getColumnInfo().equals(columnInfo)) { - dagRequestBuilder.addOutputOffsets(pos); - colOffsetInFieldMap.put(col, pos); - } - } - // if a column of field is not contained in index selected, - // logically it must be the pk column and - // the pkIsHandle must be true. Extra check here. - else if (col.getColumnInfo().isPrimaryKey() && tableInfo.isPkHandle()) { - pkIsNeeded = true; - // offset should be processed for each primary key encountered - dagRequestBuilder.addOutputOffsets(colCount); - // for index scan, column offset must be in the order of index->handle - colOffsetInFieldMap.put(col, indexColOffsets.size()); - } else { - throw new DAGRequestException( - "columns other than primary key and index key exist in fields while index single read: " - + col.getName()); - } - } - // pk is not included in index but still needed - if (pkIsNeeded) { - indexScanBuilder.addColumns(handleColumn); - } - } - executorBuilder.setTp(ExecType.TypeIndexScan); - - indexScanBuilder.setTableId(tableInfo.getId()).setIndexId(indexInfo.getId()); - dagRequestBuilder.addExecutors(executorBuilder.setIdxScan(indexScanBuilder).build()); - } else { - // TableScan - executorBuilder.setTp(ExecType.TypeTableScan); - tblScanBuilder.setTableId(tableInfo.getId()); - // Step1. Add columns to first executor - for (int i = 0; i < getFields().size(); i++) { - ColumnRef col = getFields().get(i); - tblScanBuilder.addColumns(col.getColumnInfo().toProto(tableInfo)); - colOffsetInFieldMap.put(col, i); - } - // Currently, according to TiKV's implementation, if handle - // is needed, we should add an extra column with an ID of -1 - // to the TableScan executor - if (isHandleNeeded()) { - tblScanBuilder.addColumns(handleColumn); - } - dagRequestBuilder.addExecutors(executorBuilder.setTblScan(tblScanBuilder)); - - // column offset should be in accordance with fields - for (int i = 0; i < getFields().size(); i++) { - dagRequestBuilder.addOutputOffsets(i); - } - - // if handle is needed, we should append one output offset - if (isHandleNeeded()) { - dagRequestBuilder.addOutputOffsets(tableInfo.getColumns().size()); - } - } - - if (!isIndexScan || (isIndexScan() && !isDoubleRead())) { - // clear executorBuilder - executorBuilder.clear(); - - // Step2. Add others - // DO NOT EDIT EXPRESSION CONSTRUCTION ORDER - // Or make sure the construction order is below: - // TableScan/IndexScan > Selection > Aggregation > TopN/Limit - Expression whereExpr = mergeCNFExpressions(getFilters()); - if (whereExpr != null) { - executorBuilder.setTp(ExecType.TypeSelection); - dagRequestBuilder.addExecutors( - executorBuilder.setSelection( - Selection.newBuilder() - .addConditions(ProtoConverter.toProto(whereExpr, colOffsetInFieldMap)))); - executorBuilder.clear(); - } - - if (!getGroupByItems().isEmpty() || !getAggregates().isEmpty()) { - Aggregation.Builder aggregationBuilder = Aggregation.newBuilder(); - getGroupByItems() - .forEach( - tiByItem -> - aggregationBuilder.addGroupBy( - ProtoConverter.toProto(tiByItem.getExpr(), colOffsetInFieldMap))); - getAggregates() - .forEach( - tiExpr -> - aggregationBuilder.addAggFunc( - ProtoConverter.toProto(tiExpr, colOffsetInFieldMap))); - executorBuilder.setTp(ExecType.TypeAggregation); - dagRequestBuilder.addExecutors(executorBuilder.setAggregation(aggregationBuilder)); - executorBuilder.clear(); - } - - if (!getOrderByItems().isEmpty()) { - TopN.Builder topNBuilder = TopN.newBuilder(); - getOrderByItems() - .forEach( - tiByItem -> - topNBuilder.addOrderBy( - com.pingcap.tidb.tipb.ByItem.newBuilder() - .setExpr( - ProtoConverter.toProto(tiByItem.getExpr(), colOffsetInFieldMap)) - .setDesc(tiByItem.isDesc()))); - executorBuilder.setTp(ExecType.TypeTopN); - topNBuilder.setLimit(getLimit()); - dagRequestBuilder.addExecutors(executorBuilder.setTopN(topNBuilder)); - executorBuilder.clear(); - } else if (getLimit() != 0) { - Limit.Builder limitBuilder = Limit.newBuilder(); - limitBuilder.setLimit(getLimit()); - executorBuilder.setTp(ExecType.TypeLimit); - dagRequestBuilder.addExecutors(executorBuilder.setLimit(limitBuilder)); - executorBuilder.clear(); - } - } - - DAGRequest request = - dagRequestBuilder - .setTimeZoneOffset(timeZoneOffset) - .setFlags(flags) - .setStartTs(startTs) - .build(); - - validateRequest(request); - - return request; - } - - /** - * Check if a DAG request is valid. - * - *

Note: When constructing a DAG request, a executor with an ExecType of higher priority should - * always be placed before those lower ones. - * - * @param dagRequest Request DAG. - */ - private void validateRequest(DAGRequest dagRequest) { - requireNonNull(dagRequest); - // A DAG request must has at least one executor. - if (dagRequest.getExecutorsCount() < 1) { - throw new DAGRequestException("Invalid executors count:" + dagRequest.getExecutorsCount()); - } - - ExecType formerType = dagRequest.getExecutors(0).getTp(); - if (formerType != ExecType.TypeTableScan && formerType != ExecType.TypeIndexScan) { - throw new DAGRequestException( - "Invalid first executor type:" - + formerType - + ", must one of TypeTableScan or TypeIndexScan"); - } - - for (int i = 1; i < dagRequest.getExecutorsCount(); i++) { - ExecType currentType = dagRequest.getExecutors(i).getTp(); - if (EXEC_TYPE_PRIORITY_MAP.get(currentType) < EXEC_TYPE_PRIORITY_MAP.get(formerType)) { - throw new DAGRequestException("Invalid executor priority."); - } - formerType = currentType; - } - } - - public TiDAGRequest setTableInfo(TiTableInfo tableInfo) { - this.tableInfo = requireNonNull(tableInfo, "tableInfo is null"); - return this; - } - - public TiTableInfo getTableInfo() { - return this.tableInfo; - } - - public TiDAGRequest setIndexInfo(TiIndexInfo indexInfo) { - this.indexInfo = requireNonNull(indexInfo, "indexInfo is null"); - return this; - } - - public TiIndexInfo getIndexInfo() { - return indexInfo; - } - - public void clearIndexInfo() { - indexInfo = null; - } - - public int getLimit() { - return limit; - } - - /** - * add limit clause to select query. - * - * @param limit is just a integer. - * @return a SelectBuilder - */ - public TiDAGRequest setLimit(int limit) { - this.limit = limit; - return this; - } - - /** - * set timezone offset - * - * @param timeZoneOffset timezone offset - * @return a TiDAGRequest - */ - public TiDAGRequest setTimeZoneOffset(int timeZoneOffset) { - this.timeZoneOffset = timeZoneOffset; - return this; - } - - int getTimeZoneOffset() { - return timeZoneOffset; - } - - /** - * set truncate mode - * - * @param mode truncate mode - * @return a TiDAGRequest - */ - public TiDAGRequest setTruncateMode(TiDAGRequest.TruncateMode mode) { - flags = requireNonNull(mode, "mode is null").mask(flags); - return this; - } - - @VisibleForTesting - public long getFlags() { - return flags; - } - - /** - * set start timestamp for the transaction - * - * @param startTs timestamp - * @return a TiDAGRequest - */ - public TiDAGRequest setStartTs(long startTs) { - this.startTs = startTs; - return this; - } - - long getStartTs() { - return startTs; - } - - /** - * set having clause to select query - * - * @param having is a expression represents Having - * @return a TiDAGRequest - */ - public TiDAGRequest setHaving(Expression having) { - this.having = requireNonNull(having, "having is null"); - return this; - } - - public TiDAGRequest setDistinct(boolean distinct) { - this.distinct = distinct; - return this; - } - - public boolean isDistinct() { - return distinct; - } - - public TiDAGRequest addAggregate(Expression expr, DataType targetType) { - requireNonNull(expr, "aggregation expr is null"); - aggregates.add(Pair.create(expr, targetType)); - return this; - } - - public List getAggregates() { - return aggregates.stream().map(p -> p.first).collect(Collectors.toList()); - } - - public List> getAggregatePairs() { - return aggregates; - } - - /** - * add a order by clause to select query. - * - * @param byItem is a TiByItem. - * @return a SelectBuilder - */ - public TiDAGRequest addOrderByItem(ByItem byItem) { - orderByItems.add(requireNonNull(byItem, "byItem is null")); - return this; - } - - List getOrderByItems() { - return orderByItems; - } - - /** - * add a group by clause to select query - * - * @param byItem is a TiByItem - * @return a SelectBuilder - */ - public TiDAGRequest addGroupByItem(ByItem byItem) { - groupByItems.add(requireNonNull(byItem, "byItem is null")); - return this; - } - - public List getGroupByItems() { - return groupByItems; - } - - /** - * Field is not support in TiDB yet, for here we simply allow TiColumnRef instead of TiExpr like - * in SelectRequest proto - * - *

- * - *

This interface allows duplicate columns and it's user's responsibility to do dedup since we - * need to ensure exact order and items preserved during decoding - * - * @param column is column referred during selectReq - */ - public TiDAGRequest addRequiredColumn(ColumnRef column) { - fields.add(requireNonNull(column, "columnRef is null")); - return this; - } - - public List getFields() { - return fields; - } - - /** - * set key range of scan - * - * @param ranges key range of scan - */ - public TiDAGRequest addRanges(List ranges) { - keyRanges.addAll(requireNonNull(ranges, "KeyRange is null")); - return this; - } - - public void resetFilters(List filters) { - this.filters.clear(); - this.filters.addAll(filters); - } - - public List getRanges() { - return keyRanges; - } - - public TiDAGRequest addFilter(Expression filter) { - this.filters.add(requireNonNull(filter, "filters expr is null")); - return this; - } - - public List getDowngradeFilters() { - return downgradeFilters; - } - - public TiDAGRequest addDowngradeFilter(Expression filter) { - this.downgradeFilters.add(requireNonNull(filter, "downgrade filter is null")); - return this; - } - - /** - * Check whether the DAG request has any aggregate expression. - * - * @return the boolean - */ - public boolean hasAggregate() { - return !getAggregates().isEmpty(); - } - - /** - * Check whether the DAG request has any group by expression. - * - * @return the boolean - */ - public boolean hasGroupBy() { - return !getGroupByItems().isEmpty(); - } - - public List getFilters() { - return filters; - } - - /** - * Returns whether handle is needed. - * - * @return the boolean - */ - public boolean isHandleNeeded() { - return handleNeeded; - } - - /** - * Sets handle needed. - * - * @param handleNeeded the handle needed - */ - public void setHandleNeeded(boolean handleNeeded) { - this.handleNeeded = handleNeeded; - } - - /** - * Returns whether needs double read - * - * @return boolean - */ - public boolean isDoubleRead() { - return isDoubleRead; - } - - /** - * Sets isDoubleRead - * - * @param isDoubleRead if is double read - */ - public void setIsDoubleRead(boolean isDoubleRead) { - this.isDoubleRead = isDoubleRead; - } - - /** - * Returns whether this request is of indexScanType - * - * @return true iff indexInfo is provided, false otherwise - */ - public boolean isIndexScan() { - return indexInfo != null; - } - - /** - * Whether we use streaming processing to retrieve data - * - * @return push down type. - */ - public PushDownType getPushDownType() { - return pushDownType; - } - - /** Set the estimated row count will be fetched from this request. */ - public void setEstimatedCount(double estimatedCount) { - this.estimatedCount = estimatedCount; - } - - /** Get the estimated row count will be fetched from this request. */ - public double getEstimatedCount() { - return estimatedCount; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - if (tableInfo != null) { - sb.append(String.format("[table: %s] ", tableInfo.getName())); - } - - if (indexInfo != null) { - sb.append(String.format("[Index: %s] ", indexInfo.getName())); - } - - if (!getFields().isEmpty()) { - sb.append(", Columns: "); - Joiner.on(", ").skipNulls().appendTo(sb, getFields()); - } - - if (!getDowngradeFilters().isEmpty()) { - // should be called after all parameters are set - if (pushdownFilters == null) { - pushdownFilters = new ArrayList<>(getDowngradeFilters()); - pushdownFilters.removeAll(new HashSet<>(getFilters())); - } - if (!pushdownFilters.isEmpty()) { - sb.append(", Pushdown Filter: "); - Joiner.on(", ").skipNulls().appendTo(sb, pushdownFilters); - } - } - - if (!getFilters().isEmpty()) { - sb.append(", Residual Filter: "); - Joiner.on(", ").skipNulls().appendTo(sb, getFilters()); - } - - // Key ranges might be also useful - if (!getRanges().isEmpty()) { - sb.append(", KeyRange: "); - getRanges().forEach(x -> sb.append(KeyUtils.formatBytes(x))); - } - - if (!getAggregates().isEmpty()) { - sb.append(", Aggregates: "); - Joiner.on(", ").skipNulls().appendTo(sb, getAggregates()); - } - - if (!getGroupByItems().isEmpty()) { - sb.append(", Group By: "); - Joiner.on(", ").skipNulls().appendTo(sb, getGroupByItems()); - } - - if (!getOrderByItems().isEmpty()) { - sb.append(", Order By: "); - Joiner.on(", ").skipNulls().appendTo(sb, getOrderByItems()); - } - - if (getLimit() != 0) { - sb.append(", Limit: "); - sb.append("[").append(limit).append("]"); - } - return sb.toString(); - } - - public TiDAGRequest copy() { - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream(baos); - oos.writeObject(this); - ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); - ObjectInputStream ois = new ObjectInputStream(bais); - return ((TiDAGRequest) ois.readObject()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/org/tikv/meta/TiDBInfo.java b/src/main/java/org/tikv/meta/TiDBInfo.java deleted file mode 100644 index d0fa9b2c2d..0000000000 --- a/src/main/java/org/tikv/meta/TiDBInfo.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; - -@JsonIgnoreProperties(ignoreUnknown = true) -public class TiDBInfo { - private final long id; - private final String name; - private final String charset; - private final String collate; - private final List tables; - private final SchemaState schemaState; - - @JsonCreator - public TiDBInfo( - @JsonProperty("id") long id, - @JsonProperty("db_name") CIStr name, - @JsonProperty("charset") String charset, - @JsonProperty("collate") String collate, - @JsonProperty("-") List tables, - @JsonProperty("state") int schemaState) { - this.id = id; - this.name = name.getL(); - this.charset = charset; - this.collate = collate; - this.tables = tables; - this.schemaState = SchemaState.fromValue(schemaState); - } - - private TiDBInfo( - long id, - String name, - String charset, - String collate, - List tables, - SchemaState schemaState) { - this.id = id; - this.name = name; - this.charset = charset; - this.collate = collate; - this.tables = tables; - this.schemaState = schemaState; - } - - public TiDBInfo rename(String newName) { - return new TiDBInfo(id, newName, charset, collate, tables, schemaState); - } - - public long getId() { - return id; - } - - public String getName() { - return name; - } - - public String getCharset() { - return charset; - } - - public String getCollate() { - return collate; - } - - public List getTables() { - return tables; - } - - SchemaState getSchemaState() { - return schemaState; - } - - @Override - public boolean equals(Object other) { - if (other == this) { - return true; - } - if (!(other instanceof TiDBInfo)) { - return false; - } - TiDBInfo otherDB = (TiDBInfo) other; - return otherDB.getId() == getId() && otherDB.getName().equals(getName()); - } - - @Override - public int hashCode() { - final int prime = 31; - int result = prime + Long.hashCode(getId()); - return result * prime + getName().hashCode(); - } -} diff --git a/src/main/java/org/tikv/meta/TiIndexColumn.java b/src/main/java/org/tikv/meta/TiIndexColumn.java deleted file mode 100644 index ffb8c0ba32..0000000000 --- a/src/main/java/org/tikv/meta/TiIndexColumn.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import java.io.Serializable; -import org.tikv.types.DataType; - -@JsonIgnoreProperties(ignoreUnknown = true) -public class TiIndexColumn implements Serializable { - private String name; - private int offset; - private long length; - - @JsonCreator - public TiIndexColumn( - @JsonProperty("name") CIStr name, - @JsonProperty("offset") int offset, - @JsonProperty("length") long length) { - this.name = name.getL(); - this.offset = offset; - this.length = length; - } - - public String getName() { - return name; - } - - public int getOffset() { - return offset; - } - - public long getLength() { - return length; - } - - public boolean isPrefixIndex() { - return length != DataType.UNSPECIFIED_LEN; - } - - public boolean matchName(String otherName) { - return name.equalsIgnoreCase(otherName); - } - - @Override - public String toString() { - return String.format( - "%s {name: %s, offset: %d, length: %d}", getClass().getSimpleName(), name, offset, length); - } -} diff --git a/src/main/java/org/tikv/meta/TiIndexInfo.java b/src/main/java/org/tikv/meta/TiIndexInfo.java deleted file mode 100644 index 7c65cf0ee2..0000000000 --- a/src/main/java/org/tikv/meta/TiIndexInfo.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import static java.util.Objects.requireNonNull; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.pingcap.tidb.tipb.ColumnInfo; -import com.pingcap.tidb.tipb.IndexInfo; -import java.io.Serializable; -import java.util.List; -import java.util.stream.Collectors; - -@JsonIgnoreProperties(ignoreUnknown = true) -public class TiIndexInfo implements Serializable { - private final long id; - private final String name; - private final String tableName; - private final List indexColumns; - private final boolean isUnique; - private final boolean isPrimary; - private final SchemaState schemaState; - private final String comment; - private final IndexType indexType; - private final boolean isFakePrimaryKey; - - @JsonCreator - @VisibleForTesting - public TiIndexInfo( - @JsonProperty("id") long id, - @JsonProperty("idx_name") CIStr name, - @JsonProperty("tbl_name") CIStr tableName, - @JsonProperty("idx_cols") List indexColumns, - @JsonProperty("is_unique") boolean isUnique, - @JsonProperty("is_primary") boolean isPrimary, - @JsonProperty("state") int schemaState, - @JsonProperty("comment") String comment, - @JsonProperty("index_type") int indexType, - // This is a fake property and added JsonProperty only to - // to bypass Jackson frameworks's check - @JsonProperty("___isFakePrimaryKey") boolean isFakePrimaryKey) { - this.id = id; - this.name = requireNonNull(name, "index name is null").getL(); - this.tableName = requireNonNull(tableName, "table name is null").getL(); - this.indexColumns = ImmutableList.copyOf(requireNonNull(indexColumns, "indexColumns is null")); - this.isUnique = isUnique; - this.isPrimary = isPrimary; - this.schemaState = SchemaState.fromValue(schemaState); - this.comment = comment; - this.indexType = IndexType.fromValue(indexType); - this.isFakePrimaryKey = isFakePrimaryKey; - } - - public static TiIndexInfo generateFakePrimaryKeyIndex(TiTableInfo table) { - TiColumnInfo pkColumn = table.getPrimaryKeyColumn(); - if (pkColumn != null) { - return new TiIndexInfo( - -1, - CIStr.newCIStr("fake_pk_" + table.getId()), - CIStr.newCIStr(table.getName()), - ImmutableList.of(pkColumn.toFakeIndexColumn()), - true, - true, - SchemaState.StatePublic.getStateCode(), - "Fake Column", - IndexType.IndexTypeHash.getTypeCode(), - true); - } - return null; - } - - public long getId() { - return id; - } - - public String getName() { - return name; - } - - public String getTableName() { - return tableName; - } - - public List getIndexColumns() { - return indexColumns; - } - - public boolean isUnique() { - return isUnique; - } - - public boolean isPrimary() { - return isPrimary; - } - - public SchemaState getSchemaState() { - return schemaState; - } - - public String getComment() { - return comment; - } - - public IndexType getIndexType() { - return indexType; - } - - public IndexInfo toProto(TiTableInfo tableInfo) { - IndexInfo.Builder builder = - IndexInfo.newBuilder().setTableId(tableInfo.getId()).setIndexId(id).setUnique(isUnique); - - List columns = tableInfo.getColumns(); - - for (TiIndexColumn indexColumn : getIndexColumns()) { - int offset = indexColumn.getOffset(); - TiColumnInfo column = columns.get(offset); - builder.addColumns(column.toProto(tableInfo)); - } - - if (tableInfo.isPkHandle()) { - for (TiColumnInfo column : columns) { - if (!column.isPrimaryKey()) { - continue; - } - ColumnInfo pbColumn = column.toProtoBuilder(tableInfo).setPkHandle(true).build(); - builder.addColumns(pbColumn); - } - } - return builder.build(); - } - - public boolean isFakePrimaryKey() { - return isFakePrimaryKey; - } - - @Override - public String toString() { - return String.format( - "%s[%s]", - name, - Joiner.on(",") - .skipNulls() - .join( - indexColumns - .stream() - .map(column -> column.getName()) - .collect(Collectors.toList()))); - } -} diff --git a/src/main/java/org/tikv/meta/TiPartitionDef.java b/src/main/java/org/tikv/meta/TiPartitionDef.java deleted file mode 100644 index 125eae89e8..0000000000 --- a/src/main/java/org/tikv/meta/TiPartitionDef.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.tikv.meta; - -import com.google.common.annotations.VisibleForTesting; -import java.io.Serializable; -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; - -@JsonIgnoreProperties(ignoreUnknown = true) -public class TiPartitionDef implements Serializable { - private final long id; - private final CIStr name; - private final String[] lessThan; - private final String comment; - - @JsonCreator - @VisibleForTesting - public TiPartitionDef( - @JsonProperty("id") long id, - @JsonProperty("name") CIStr name, - @JsonProperty("less_than") String[] lessThan, - @JsonProperty("comment") String comment) { - this.id = id; - this.name = name; - this.lessThan = lessThan; - this.comment = comment; - } -} diff --git a/src/main/java/org/tikv/meta/TiPartitionInfo.java b/src/main/java/org/tikv/meta/TiPartitionInfo.java deleted file mode 100644 index 0994095e0e..0000000000 --- a/src/main/java/org/tikv/meta/TiPartitionInfo.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.tikv.meta; - -import com.google.common.annotations.VisibleForTesting; -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; - -@JsonIgnoreProperties(ignoreUnknown = true) -public class TiPartitionInfo { - public static enum PartitionType { - RangePartition, - HashPartition, - ListPartition, - } - - private final PartitionType type; - private final String expr; - private final CIStr[] columns; - private final boolean enable; - private final TiPartitionDef[] defs; - - @JsonCreator - @VisibleForTesting - public TiPartitionInfo( - @JsonProperty("type") PartitionType type, - @JsonProperty("expr") String expr, - @JsonProperty("columns") CIStr[] columns, - @JsonProperty("enable") boolean enable, - @JsonProperty("definitions") TiPartitionDef[] defs) { - this.type = type; - this.expr = expr; - this.columns = columns; - this.enable = enable; - this.defs = defs; - } -} diff --git a/src/main/java/org/tikv/meta/TiTableInfo.java b/src/main/java/org/tikv/meta/TiTableInfo.java deleted file mode 100644 index 56502b305e..0000000000 --- a/src/main/java/org/tikv/meta/TiTableInfo.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import static java.util.Objects.requireNonNull; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.pingcap.tidb.tipb.TableInfo; -import java.io.Serializable; -import java.util.List; -import java.util.stream.Collectors; -import org.tikv.exception.TiClientInternalException; -import org.tikv.meta.TiColumnInfo.InternalTypeHolder; -import org.tikv.types.DataType; -import org.tikv.types.DataTypeFactory; - -@JsonIgnoreProperties(ignoreUnknown = true) -public class TiTableInfo implements Serializable { - private final long id; - private final String name; - private final String charset; - private final String collate; - private final List columns; - private final List indices; - private final boolean pkIsHandle; - private final String comment; - private final long autoIncId; - private final long maxColumnId; - private final long maxIndexId; - private final long oldSchemaId; - private final TiPartitionInfo partitionInfo; - - @JsonCreator - public TiTableInfo( - @JsonProperty("id") long id, - @JsonProperty("name") CIStr name, - @JsonProperty("charset") String charset, - @JsonProperty("collate") String collate, - @JsonProperty("pk_is_handle") boolean pkIsHandle, - @JsonProperty("cols") List columns, - @JsonProperty("index_info") List indices, - @JsonProperty("comment") String comment, - @JsonProperty("auto_inc_id") long autoIncId, - @JsonProperty("max_col_id") long maxColumnId, - @JsonProperty("max_idx_id") long maxIndexId, - @JsonProperty("old_schema_id") long oldSchemaId, - @JsonProperty("partition") TiPartitionInfo partitionInfo) { - this.id = id; - this.name = name.getL(); - this.charset = charset; - this.collate = collate; - this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); - this.pkIsHandle = pkIsHandle; - this.indices = indices != null ? ImmutableList.copyOf(indices) : ImmutableList.of(); - this.comment = comment; - this.autoIncId = autoIncId; - this.maxColumnId = maxColumnId; - this.maxIndexId = maxIndexId; - this.oldSchemaId = oldSchemaId; - this.partitionInfo = partitionInfo; - } - - public long getId() { - return id; - } - - public String getName() { - return name; - } - - public String getCharset() { - return charset; - } - - public String getCollate() { - return collate; - } - - public List getColumns() { - return columns; - } - - public TiColumnInfo getColumn(int offset) { - if (offset < 0 || offset >= columns.size()) { - throw new TiClientInternalException(String.format("Column offset %d out of bound", offset)); - } - return columns.get(offset); - } - - public boolean isPkHandle() { - return pkIsHandle; - } - - public List getIndices() { - return indices; - } - - public String getComment() { - return comment; - } - - public long getAutoIncId() { - return autoIncId; - } - - public long getMaxColumnId() { - return maxColumnId; - } - - public long getMaxIndexId() { - return maxIndexId; - } - - public long getOldSchemaId() { - return oldSchemaId; - } - - public TableInfo toProto() { - return TableInfo.newBuilder() - .setTableId(getId()) - .addAllColumns( - getColumns().stream().map(col -> col.toProto(this)).collect(Collectors.toList())) - .build(); - } - - // Only Integer Column will be a PK column - // and there exists only one PK column - public TiColumnInfo getPrimaryKeyColumn() { - if (isPkHandle()) { - for (TiColumnInfo col : getColumns()) { - if (col.isPrimaryKey()) { - return col; - } - } - } - return null; - } - - public TiTableInfo copyTableWithRowId() { - if (!isPkHandle()) { - ImmutableList.Builder newColumns = ImmutableList.builder(); - for (TiColumnInfo col : getColumns()) { - DataType type = col.getType(); - InternalTypeHolder typeHolder = type.toTypeHolder(); - typeHolder.setFlag(type.getFlag() & (~DataType.PriKeyFlag)); - DataType newType = DataTypeFactory.of(typeHolder); - TiColumnInfo newCol = - new TiColumnInfo( - col.getId(), - col.getName(), - col.getOffset(), - newType, - col.getSchemaState(), - col.getOriginDefaultValue(), - col.getDefaultValue(), - col.getComment()); - newColumns.add(newCol.copyWithoutPrimaryKey()); - } - newColumns.add(TiColumnInfo.getRowIdColumn(getColumns().size())); - return new TiTableInfo( - getId(), - CIStr.newCIStr(getName()), - getCharset(), - getCollate(), - true, - newColumns.build(), - getIndices(), - getComment(), - getAutoIncId(), - getMaxColumnId(), - getMaxIndexId(), - getOldSchemaId(), - partitionInfo); - } else { - return this; - } - } - - @Override - public String toString() { - return toProto().toString(); - } -} diff --git a/src/main/java/org/tikv/operation/KVErrorHandler.java b/src/main/java/org/tikv/operation/KVErrorHandler.java index 5e4a6566b3..ad5e8cbe4a 100644 --- a/src/main/java/org/tikv/operation/KVErrorHandler.java +++ b/src/main/java/org/tikv/operation/KVErrorHandler.java @@ -23,7 +23,6 @@ import io.grpc.StatusRuntimeException; import java.util.function.Function; import org.apache.log4j.Logger; import org.tikv.codec.KeyUtils; -import org.tikv.event.CacheInvalidateEvent; import org.tikv.exception.GrpcException; import org.tikv.kvproto.Errorpb; import org.tikv.region.RegionErrorReceiver; @@ -38,7 +37,6 @@ public class KVErrorHandler implements ErrorHandler { private static final int NO_LEADER_STORE_ID = 0; // if there's currently no leader of a store, store id is set to 0 private final Function getRegionError; - private final Function cacheInvalidateCallBack; private final RegionManager regionManager; private final RegionErrorReceiver recv; private final TiRegion ctxRegion; @@ -52,10 +50,6 @@ public class KVErrorHandler implements ErrorHandler { this.recv = recv; this.regionManager = regionManager; this.getRegionError = getRegionError; - this.cacheInvalidateCallBack = - regionManager != null && regionManager.getSession() != null - ? regionManager.getSession().getCacheInvalidateCallback() - : null; } private Errorpb.Error getRegionError(RespT resp) { @@ -68,55 +62,6 @@ public class KVErrorHandler implements ErrorHandler { private void invalidateRegionStoreCache(TiRegion ctxRegion) { regionManager.invalidateRegion(ctxRegion.getId()); regionManager.invalidateStore(ctxRegion.getLeader().getStoreId()); - notifyRegionStoreCacheInvalidate( - ctxRegion.getId(), - ctxRegion.getLeader().getStoreId(), - CacheInvalidateEvent.CacheType.REGION_STORE); - } - - /** Used for notifying Spark driver to invalidate cache from Spark workers. */ - private void notifyRegionStoreCacheInvalidate( - long regionId, long storeId, CacheInvalidateEvent.CacheType type) { - if (cacheInvalidateCallBack != null) { - cacheInvalidateCallBack.apply(new CacheInvalidateEvent(regionId, storeId, true, true, type)); - logger.info( - "Accumulating cache invalidation info to driver:regionId=" - + regionId - + ",storeId=" - + storeId - + ",type=" - + type.name()); - } else { - logger.warn( - "Failed to send notification back to driver since CacheInvalidateCallBack is null in executor node."); - } - } - - private void notifyRegionCacheInvalidate(long regionId) { - if (cacheInvalidateCallBack != null) { - cacheInvalidateCallBack.apply( - new CacheInvalidateEvent( - regionId, 0, true, false, CacheInvalidateEvent.CacheType.REGION_STORE)); - logger.info( - "Accumulating cache invalidation info to driver:regionId=" - + regionId - + ",type=" - + CacheInvalidateEvent.CacheType.REGION_STORE.name()); - } else { - logger.warn( - "Failed to send notification back to driver since CacheInvalidateCallBack is null in executor node."); - } - } - - private void notifyStoreCacheInvalidate(long storeId) { - if (cacheInvalidateCallBack != null) { - cacheInvalidateCallBack.apply( - new CacheInvalidateEvent( - 0, storeId, false, true, CacheInvalidateEvent.CacheType.REGION_STORE)); - } else { - logger.warn( - "Failed to send notification back to driver since CacheInvalidateCallBack is null in executor node."); - } } // Referenced from TiDB @@ -161,9 +106,6 @@ public class KVErrorHandler implements ErrorHandler { // to a new store address. retry = false; } - notifyRegionStoreCacheInvalidate( - ctxRegion.getId(), newStoreId, CacheInvalidateEvent.CacheType.LEADER); - backOffFuncType = BackOffFunction.BackOffFuncType.BoUpdateLeader; } else { logger.info( @@ -171,9 +113,7 @@ public class KVErrorHandler implements ErrorHandler { "Received zero store id, from region %d try next time", ctxRegion.getId())); backOffFuncType = BackOffFunction.BackOffFuncType.BoRegionMiss; } - backOffer.doBackOff(backOffFuncType, new GrpcException(error.toString())); - return retry; } else if (error.hasStoreNotMatch()) { // this error is reported from raftstore: @@ -184,17 +124,18 @@ public class KVErrorHandler implements ErrorHandler { String.format( "Store Not Match happened with region id %d, store id %d", ctxRegion.getId(), storeId)); + logger.warn(String.format("%s", error.getStoreNotMatch())); this.regionManager.invalidateStore(storeId); recv.onStoreNotMatch(this.regionManager.getStoreById(storeId)); - notifyStoreCacheInvalidate(storeId); + backOffer.doBackOff( + BackOffFunction.BackOffFuncType.BoStoreNotMatch, new GrpcException(error.toString())); return true; } else if (error.hasStaleEpoch()) { // this error is reported from raftstore: // region has outdated version，please try later. logger.warn(String.format("Stale Epoch encountered for region [%s]", ctxRegion)); this.regionManager.onRegionStale(ctxRegion.getId()); - notifyRegionCacheInvalidate(ctxRegion.getId()); return false; } else if (error.hasServerIsBusy()) { // this error is reported from kv: @@ -241,10 +182,6 @@ public class KVErrorHandler implements ErrorHandler { @Override public boolean handleRequestError(BackOffer backOffer, Exception e) { regionManager.onRequestFail(ctxRegion.getId(), ctxRegion.getLeader().getStoreId()); - notifyRegionStoreCacheInvalidate( - ctxRegion.getId(), - ctxRegion.getLeader().getStoreId(), - CacheInvalidateEvent.CacheType.REQ_FAILED); backOffer.doBackOff( BackOffFunction.BackOffFuncType.BoTiKVRPC, diff --git a/src/main/java/org/tikv/operation/SchemaInfer.java b/src/main/java/org/tikv/operation/SchemaInfer.java deleted file mode 100644 index 10f66629c4..0000000000 --- a/src/main/java/org/tikv/operation/SchemaInfer.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.operation; - -import java.util.ArrayList; -import java.util.List; -import org.tikv.expression.ByItem; -import org.tikv.expression.Expression; -import org.tikv.meta.TiDAGRequest; -import org.tikv.operation.transformer.Cast; -import org.tikv.operation.transformer.NoOp; -import org.tikv.operation.transformer.RowTransformer; -import org.tikv.types.DataType; -import org.tikv.types.IntegerType; -import org.tikv.util.Pair; - -/** - * SchemaInfer extract row's type after query is executed. It is pretty rough version. Optimization - * is on the way. The problem we have right now is that TiDB promote Sum to Decimal which is not - * compatible with column's type. The solution we come up with right now is use record column's type - * ad finalFieldType and build another list recording TiExpr's type as fieldType for row reading. - * Once we finish row reading, we first check each element in fieldType and finalFieldType share the - * same type or not. If yes, no need for casting. If no, casting is needed here. - */ -public class SchemaInfer { - private List types; - private RowTransformer rt; - - public static SchemaInfer create(TiDAGRequest dagRequest) { - return new SchemaInfer(dagRequest); - } - - protected SchemaInfer(TiDAGRequest dagRequest) { - types = new ArrayList<>(); - extractFieldTypes(dagRequest); - extractHandleType(dagRequest); - buildTransform(dagRequest); - } - - private void extractHandleType(TiDAGRequest dagRequest) { - if (dagRequest.isHandleNeeded()) { - // DataType of handle is long - types.add(IntegerType.INT); - } - } - - private void buildTransform(TiDAGRequest dagRequest) { - RowTransformer.Builder rowTrans = RowTransformer.newBuilder(); - // Update: - // Switching to DAG mode will eliminate first blob - // TODO:check correctness of ↑ - // 1. if group by is empty, first column should be "single group" - // which is a string - // 2. if multiple group by items present, it is wrapped inside - // a byte array. we make a multiple decoding - // 3. for no aggregation case, make only projected columns - - // append aggregates if present - if (dagRequest.hasAggregate()) { - for (Pair pair : dagRequest.getAggregatePairs()) { - rowTrans.addProjection(new Cast(pair.second)); - } - if (dagRequest.hasGroupBy()) { - for (ByItem byItem : dagRequest.getGroupByItems()) { - rowTrans.addProjection(new NoOp(dagRequest.getExpressionType(byItem.getExpr()))); - } - } - } else { - for (Expression field : dagRequest.getFields()) { - rowTrans.addProjection(new NoOp(dagRequest.getExpressionType(field))); - } - } - rowTrans.addSourceFieldTypes(types); - rt = rowTrans.build(); - } - - /** - * TODO: order by extract field types from tiSelectRequest for reading data to row. - * - * @param dagRequest is SelectRequest - */ - private void extractFieldTypes(TiDAGRequest dagRequest) { - if (dagRequest.hasAggregate()) { - dagRequest.getAggregates().forEach(expr -> types.add(dagRequest.getExpressionType(expr))); - // In DAG mode, if there is any group by statement in a request, all the columns specified - // in group by expression will be returned, so when we decode a result row, we need to pay - // extra attention to decoding. - if (dagRequest.hasGroupBy()) { - for (ByItem item : dagRequest.getGroupByItems()) { - types.add(dagRequest.getExpressionType(item.getExpr())); - } - } - } else { - // Extract all column type information from TiExpr - dagRequest.getFields().forEach(expr -> types.add(expr.getType())); - } - } - - public DataType getType(int index) { - return types.get(index); - } - - public List getTypes() { - return types; - } - - public RowTransformer getRowTransformer() { - return this.rt; - } -} diff --git a/src/main/java/org/tikv/operation/iterator/ChunkIterator.java b/src/main/java/org/tikv/operation/iterator/ChunkIterator.java deleted file mode 100644 index 2dae7f6a97..0000000000 --- a/src/main/java/org/tikv/operation/iterator/ChunkIterator.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.operation.iterator; - -import com.google.protobuf.ByteString; -import com.pingcap.tidb.tipb.Chunk; -import java.util.Iterator; -import java.util.List; -import org.tikv.exception.TiClientInternalException; - -public abstract class ChunkIterator implements Iterator { - - private final List chunks; - protected int chunkIndex; - protected int metaIndex; - protected int bufOffset; - protected boolean eof; - - public static ChunkIterator getRawBytesChunkIterator(List chunks) { - return new ChunkIterator(chunks) { - @Override - public ByteString next() { - Chunk c = chunks.get(chunkIndex); - long endOffset = c.getRowsMeta(metaIndex).getLength() + bufOffset; - if (endOffset > Integer.MAX_VALUE) { - throw new TiClientInternalException("Offset exceeded MAX_INT."); - } - - ByteString result = c.getRowsData().substring(bufOffset, (int) endOffset); - advance(); - return result; - } - }; - } - - public static ChunkIterator getHandleChunkIterator(List chunks) { - return new ChunkIterator(chunks) { - @Override - public Long next() { - Chunk c = chunks.get(chunkIndex); - long result = c.getRowsMeta(metaIndex).getHandle(); - advance(); - return result; - } - }; - } - - protected ChunkIterator(List chunks) { - // Read and then advance semantics - this.chunks = chunks; - this.chunkIndex = 0; - this.metaIndex = 0; - this.bufOffset = 0; - if (chunks.size() == 0 - || chunks.get(0).getRowsMetaCount() == 0 - || chunks.get(0).getRowsData().size() == 0) { - eof = true; - } - } - - @Override - public boolean hasNext() { - return !eof; - } - - private boolean seekNextNonEmptyChunk() { - // loop until the end of chunk list or first non empty chunk - do { - chunkIndex += 1; - } while (chunkIndex < chunks.size() && chunks.get(chunkIndex).getRowsMetaCount() == 0); - // return if remaining things left - return chunkIndex < chunks.size(); - } - - protected void advance() { - if (eof) { - return; - } - Chunk c = chunks.get(chunkIndex); - bufOffset += c.getRowsMeta(metaIndex++).getLength(); - if (metaIndex >= c.getRowsMetaCount()) { - if (seekNextNonEmptyChunk()) { - metaIndex = 0; - bufOffset = 0; - } else { - eof = true; - } - } - } -} diff --git a/src/main/java/org/tikv/operation/iterator/CoprocessIterator.java b/src/main/java/org/tikv/operation/iterator/CoprocessIterator.java deleted file mode 100644 index 7bec7bef51..0000000000 --- a/src/main/java/org/tikv/operation/iterator/CoprocessIterator.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.operation.iterator; - -import static java.util.Objects.requireNonNull; - -import com.pingcap.tidb.tipb.Chunk; -import com.pingcap.tidb.tipb.DAGRequest; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; -import org.tikv.TiSession; -import org.tikv.codec.CodecDataInput; -import org.tikv.meta.TiDAGRequest; -import org.tikv.operation.SchemaInfer; -import org.tikv.row.Row; -import org.tikv.row.RowReader; -import org.tikv.row.RowReaderFactory; -import org.tikv.types.DataType; -import org.tikv.types.IntegerType; -import org.tikv.util.RangeSplitter.RegionTask; - -public abstract class CoprocessIterator implements Iterator { - protected final TiSession session; - protected final List regionTasks; - protected DAGRequest dagRequest; - protected static final DataType[] handleTypes = new DataType[] {IntegerType.INT}; - // protected final ExecutorCompletionService> completionService; - protected RowReader rowReader; - protected CodecDataInput dataInput; - protected boolean eof = false; - protected int taskIndex; - protected int chunkIndex; - protected List chunkList; - protected SchemaInfer schemaInfer; - - CoprocessIterator( - DAGRequest req, List regionTasks, TiSession session, SchemaInfer infer) { - this.dagRequest = req; - this.session = session; - this.regionTasks = regionTasks; - this.schemaInfer = infer; - } - - abstract void submitTasks(); - - public static CoprocessIterator getRowIterator( - TiDAGRequest req, List regionTasks, TiSession session) { - return new DAGIterator( - req.buildScan(req.isIndexScan() && !req.isDoubleRead()), - regionTasks, - session, - SchemaInfer.create(req), - req.getPushDownType()) { - @Override - public Row next() { - if (hasNext()) { - return rowReader.readRow(schemaInfer.getTypes().toArray(new DataType[0])); - } else { - throw new NoSuchElementException(); - } - } - }; - } - - public static CoprocessIterator getHandleIterator( - TiDAGRequest req, List regionTasks, TiSession session) { - return new DAGIterator( - req.buildScan(true), regionTasks, session, SchemaInfer.create(req), req.getPushDownType()) { - @Override - public Long next() { - if (hasNext()) { - return rowReader.readRow(handleTypes).getLong(0); - } else { - throw new NoSuchElementException(); - } - } - }; - } - - boolean tryAdvanceChunkIndex() { - if (chunkList == null || chunkIndex >= chunkList.size() - 1) { - return false; - } - - chunkIndex++; - return true; - } - - void createDataInputReader() { - requireNonNull(chunkList, "Chunk list should not be null."); - if (0 > chunkIndex || chunkIndex >= chunkList.size()) { - throw new IllegalArgumentException(); - } - dataInput = new CodecDataInput(chunkList.get(chunkIndex).getRowsData()); - rowReader = RowReaderFactory.createRowReader(dataInput); - } -} diff --git a/src/main/java/org/tikv/operation/iterator/DAGIterator.java b/src/main/java/org/tikv/operation/iterator/DAGIterator.java deleted file mode 100644 index e83085d1a5..0000000000 --- a/src/main/java/org/tikv/operation/iterator/DAGIterator.java +++ /dev/null @@ -1,220 +0,0 @@ -package org.tikv.operation.iterator; - -import static org.tikv.meta.TiDAGRequest.PushDownType.STREAMING; - -import com.pingcap.tidb.tipb.Chunk; -import com.pingcap.tidb.tipb.DAGRequest; -import com.pingcap.tidb.tipb.SelectResponse; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Queue; -import java.util.concurrent.ExecutorCompletionService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.tikv.TiSession; -import org.tikv.exception.RegionTaskException; -import org.tikv.exception.TiClientInternalException; -import org.tikv.kvproto.Coprocessor; -import org.tikv.kvproto.Metapb; -import org.tikv.meta.TiDAGRequest.PushDownType; -import org.tikv.operation.SchemaInfer; -import org.tikv.region.RegionStoreClient; -import org.tikv.region.TiRegion; -import org.tikv.util.BackOffer; -import org.tikv.util.ConcreteBackOffer; -import org.tikv.util.RangeSplitter; - -public abstract class DAGIterator extends CoprocessIterator { - private ExecutorCompletionService> streamingService; - private ExecutorCompletionService dagService; - private SelectResponse response; - private static final Logger logger = LoggerFactory.getLogger(DAGIterator.class.getName()); - - private Iterator responseIterator; - - private final PushDownType pushDownType; - - DAGIterator( - DAGRequest req, - List regionTasks, - TiSession session, - SchemaInfer infer, - PushDownType pushDownType) { - super(req, regionTasks, session, infer); - this.pushDownType = pushDownType; - switch (pushDownType) { - case NORMAL: - dagService = new ExecutorCompletionService<>(session.getThreadPoolForTableScan()); - break; - case STREAMING: - streamingService = new ExecutorCompletionService<>(session.getThreadPoolForTableScan()); - break; - } - submitTasks(); - } - - @Override - void submitTasks() { - for (RangeSplitter.RegionTask task : regionTasks) { - switch (pushDownType) { - case STREAMING: - streamingService.submit(() -> processByStreaming(task)); - break; - case NORMAL: - dagService.submit(() -> process(task)); - break; - } - } - } - - @Override - public boolean hasNext() { - if (eof) { - return false; - } - - while (chunkList == null || chunkIndex >= chunkList.size() || dataInput.available() <= 0) { - // First we check if our chunk list has remaining chunk - if (tryAdvanceChunkIndex()) { - createDataInputReader(); - } - // If not, check next region/response - else if (pushDownType == STREAMING) { - if (!advanceNextResponse() && !readNextRegionChunks()) { - return false; - } - } else if (!readNextRegionChunks()) { - return false; - } - } - - return true; - } - - private boolean hasMoreResponse() { - switch (pushDownType) { - case STREAMING: - return responseIterator != null && responseIterator.hasNext(); - case NORMAL: - return response != null; - } - - throw new IllegalArgumentException("Invalid push down type:" + pushDownType); - } - - private boolean advanceNextResponse() { - if (!hasMoreResponse()) { - return false; - } - - switch (pushDownType) { - case STREAMING: - chunkList = responseIterator.next().getChunksList(); - break; - case NORMAL: - chunkList = response.getChunksList(); - break; - } - - if (chunkList == null || chunkList.isEmpty()) { - return false; - } - - chunkIndex = 0; - createDataInputReader(); - return true; - } - - private boolean readNextRegionChunks() { - if (eof || regionTasks == null || taskIndex >= regionTasks.size()) { - return false; - } - - try { - switch (pushDownType) { - case STREAMING: - responseIterator = streamingService.take().get(); - break; - case NORMAL: - response = dagService.take().get(); - break; - } - - } catch (Exception e) { - throw new TiClientInternalException("Error reading region:", e); - } - - taskIndex++; - return advanceNextResponse(); - } - - private SelectResponse process(RangeSplitter.RegionTask regionTask) { - Queue remainTasks = new ArrayDeque<>(); - Queue responseQueue = new ArrayDeque<>(); - remainTasks.add(regionTask); - BackOffer backOffer = ConcreteBackOffer.newCopNextMaxBackOff(); - // In case of one region task spilt into several others, we ues a queue to properly handle all - // the remaining tasks. - while (!remainTasks.isEmpty()) { - RangeSplitter.RegionTask task = remainTasks.poll(); - if (task == null) continue; - List ranges = task.getRanges(); - TiRegion region = task.getRegion(); - Metapb.Store store = task.getStore(); - - try { - RegionStoreClient client = RegionStoreClient.create(region, store, session); - Collection tasks = - client.coprocess(backOffer, dagRequest, ranges, responseQueue); - if (tasks != null) { - remainTasks.addAll(tasks); - } - } catch (Throwable e) { - // Handle region task failed - logger.error( - "Process region tasks failed, remain " - + remainTasks.size() - + " tasks not executed due to", - e); - // Rethrow to upper levels - eof = true; - throw new RegionTaskException("Handle region task failed:", e); - } - } - - // Add all chunks to the final result - List resultChunk = new ArrayList<>(); - while (!responseQueue.isEmpty()) { - SelectResponse response = responseQueue.poll(); - if (response != null) { - resultChunk.addAll(response.getChunksList()); - } - } - - return SelectResponse.newBuilder().addAllChunks(resultChunk).build(); - } - - private Iterator processByStreaming(RangeSplitter.RegionTask regionTask) { - List ranges = regionTask.getRanges(); - TiRegion region = regionTask.getRegion(); - Metapb.Store store = regionTask.getStore(); - - RegionStoreClient client; - try { - client = RegionStoreClient.create(region, store, session); - Iterator responseIterator = client.coprocessStreaming(dagRequest, ranges); - if (responseIterator == null) { - eof = true; - return null; - } - return responseIterator; - } catch (Exception e) { - // TODO: Fix stale error handling in streaming - // see:https://github.com/pingcap/tikv-client-lib-java/pull/149 - throw new TiClientInternalException("Error Closing Store client.", e); - } - } -} diff --git a/src/main/java/org/tikv/operation/iterator/IndexScanIterator.java b/src/main/java/org/tikv/operation/iterator/IndexScanIterator.java deleted file mode 100644 index c31ec2a2c9..0000000000 --- a/src/main/java/org/tikv/operation/iterator/IndexScanIterator.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.operation.iterator; - -import gnu.trove.list.array.TLongArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.concurrent.ExecutorCompletionService; -import org.tikv.Snapshot; -import org.tikv.TiConfiguration; -import org.tikv.TiSession; -import org.tikv.exception.TiClientInternalException; -import org.tikv.meta.TiDAGRequest; -import org.tikv.row.Row; -import org.tikv.util.RangeSplitter; -import org.tikv.util.RangeSplitter.RegionTask; - -public class IndexScanIterator implements Iterator { - private final Iterator handleIterator; - private final TiDAGRequest dagReq; - private final Snapshot snapshot; - private Iterator rowIterator; - private final ExecutorCompletionService> completionService; - - private int batchCount = 0; - private final int batchSize; - - public IndexScanIterator(Snapshot snapshot, TiDAGRequest req, Iterator handleIterator) { - TiSession session = snapshot.getSession(); - TiConfiguration conf = session.getConf(); - this.dagReq = req; - this.handleIterator = handleIterator; - this.snapshot = snapshot; - this.batchSize = conf.getIndexScanBatchSize(); - this.completionService = new ExecutorCompletionService<>(session.getThreadPoolForIndexScan()); - } - - private TLongArrayList feedBatch() { - TLongArrayList handles = new TLongArrayList(512); - while (handleIterator.hasNext()) { - handles.add(handleIterator.next()); - if (batchSize <= handles.size()) { - break; - } - } - return handles; - } - - @Override - public boolean hasNext() { - try { - if (rowIterator == null) { - TiSession session = snapshot.getSession(); - while (handleIterator.hasNext()) { - TLongArrayList handles = feedBatch(); - batchCount++; - completionService.submit( - () -> { - List tasks = - RangeSplitter.newSplitter(session.getRegionManager()) - .splitAndSortHandlesByRegion(dagReq.getTableInfo().getId(), handles); - return CoprocessIterator.getRowIterator(dagReq, tasks, session); - }); - } - while (batchCount > 0) { - rowIterator = completionService.take().get(); - batchCount--; - - if (rowIterator.hasNext()) { - return true; - } - } - } - if (rowIterator == null) { - return false; - } - } catch (Exception e) { - throw new TiClientInternalException("Error reading rows from handle", e); - } - return rowIterator.hasNext(); - } - - @Override - public Row next() { - if (hasNext()) { - return rowIterator.next(); - } else { - throw new NoSuchElementException(); - } - } -} diff --git a/src/main/java/org/tikv/operation/transformer/Cast.java b/src/main/java/org/tikv/operation/transformer/Cast.java deleted file mode 100644 index c405f647cc..0000000000 --- a/src/main/java/org/tikv/operation/transformer/Cast.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.operation.transformer; - -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import org.tikv.row.Row; -import org.tikv.types.*; - -public class Cast extends NoOp { - public Cast(DataType type) { - super(type); - } - - @Override - public void set(Object value, Row row, int pos) { - Object casted; - if (value == null) { - row.set(pos, targetDataType, null); - return; - } - if (targetDataType instanceof IntegerType) { - casted = castToLong(value); - } else if (targetDataType instanceof StringType) { - casted = castToString(value); - } else if (targetDataType instanceof BytesType) { - casted = castToBinary(value); - } else if (targetDataType instanceof DecimalType) { - casted = castToDecimal(value); - } else if (targetDataType instanceof RealType) { - casted = castToDouble(value); - } else { - casted = value; - } - row.set(pos, targetDataType, casted); - } - - private Double castToDouble(Object obj) { - if (obj instanceof Number) { - Number num = (Number) obj; - return num.doubleValue(); - } - throw new UnsupportedOperationException("can not cast un-number to double "); - } - - private BigDecimal castToDecimal(Object obj) { - if (obj instanceof Number) { - Number num = (Number) obj; - return new BigDecimal(num.doubleValue()); - } - throw new UnsupportedOperationException( - "Cannot cast to BigDecimal: " + (obj == null ? "null" : obj.getClass().getSimpleName())); - } - - private Long castToLong(Object obj) { - if (obj instanceof Number) { - Number num = (Number) obj; - return num.longValue(); - } - throw new UnsupportedOperationException("can not cast un-number to long "); - } - - private String castToString(Object obj) { - String result; - if (obj instanceof byte[]) { - result = new String((byte[]) obj, StandardCharsets.UTF_8); - } else if (obj instanceof char[]) { - result = new String((char[]) obj); - } else { - result = String.valueOf(obj); - } - return result; - } - - private byte[] castToBinary(Object obj) { - if (obj instanceof byte[]) { - return (byte[]) obj; - } else { - return obj.toString().getBytes(); - } - } -} diff --git a/src/main/java/org/tikv/operation/transformer/MultiKeyDecoder.java b/src/main/java/org/tikv/operation/transformer/MultiKeyDecoder.java deleted file mode 100644 index 9e35c1e466..0000000000 --- a/src/main/java/org/tikv/operation/transformer/MultiKeyDecoder.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.operation.transformer; - -import static java.util.Objects.requireNonNull; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import org.tikv.codec.CodecDataInput; -import org.tikv.row.Row; -import org.tikv.types.DataType; - -public class MultiKeyDecoder implements Projection { - public MultiKeyDecoder(List dataTypes) { - this.resultTypes = requireNonNull(dataTypes).toArray(new DataType[0]); - } - - private DataType[] resultTypes; - - @Override - public void set(Object value, Row row, int pos) { - byte[] rowData = (byte[]) value; - CodecDataInput cdi = new CodecDataInput(rowData); - - for (int i = 0; i < resultTypes.length; i++) { - DataType type = resultTypes[i]; - if (type.isNextNull(cdi)) { - cdi.readUnsignedByte(); - row.setNull(i + pos); - } else { - row.set(i + pos, type, type.decode(cdi)); - } - } - } - - @Override - public int size() { - return resultTypes.length; - } - - @Override - public List getTypes() { - return ImmutableList.copyOf(resultTypes); - } -} diff --git a/src/main/java/org/tikv/operation/transformer/NoOp.java b/src/main/java/org/tikv/operation/transformer/NoOp.java deleted file mode 100644 index 089f215419..0000000000 --- a/src/main/java/org/tikv/operation/transformer/NoOp.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.operation.transformer; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import org.tikv.row.Row; -import org.tikv.types.DataType; - -/** Noop is a base type projection, it basically do nothing but copy. */ -public class NoOp implements Projection { - protected DataType targetDataType; - - public NoOp(DataType dataType) { - this.targetDataType = dataType; - } - - @Override - public void set(Object value, Row row, int pos) { - row.set(pos, targetDataType, value); - } - - @Override - public int size() { - return 1; - } - - @Override - public List getTypes() { - return ImmutableList.of(targetDataType); - } -} diff --git a/src/main/java/org/tikv/operation/transformer/Projection.java b/src/main/java/org/tikv/operation/transformer/Projection.java deleted file mode 100644 index 4b80774e18..0000000000 --- a/src/main/java/org/tikv/operation/transformer/Projection.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.operation.transformer; - -import java.util.List; -import org.tikv.row.Row; -import org.tikv.types.DataType; - -public interface Projection { - void set(Object value, Row row, int pos); - - int size(); - - List getTypes(); -} diff --git a/src/main/java/org/tikv/operation/transformer/RowTransformer.java b/src/main/java/org/tikv/operation/transformer/RowTransformer.java deleted file mode 100644 index c3f3de16ab..0000000000 --- a/src/main/java/org/tikv/operation/transformer/RowTransformer.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.operation.transformer; - -import static java.util.Objects.requireNonNull; - -import com.google.common.collect.ImmutableList; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import org.tikv.row.ObjectRowImpl; -import org.tikv.row.Row; -import org.tikv.types.DataType; - -/** - * RowTransformer is used along with SchemaInfer and row and provide some operation. If you do not - * know your target FieldType, then you do not need use this interface. The reason we provide this - * interface is that sometime we need use it. Suppose we have a table t1 and have two column c1 and - * s1 select sum(c1) from t1 will return SingleGroup literally and sum(c1). SingleGroup should be - * skipped. Hence, skip operation is needed here. Another usage is that sum(c1)'s type is decimal no - * matter what real column type is. We need cast it to target type which is column's type. Hence, - * cast operation is needed. RowTransformer is executed after row is already read from - * CodecDataInput. - */ -public class RowTransformer { - public static Builder newBuilder() { - return new Builder(); - } - - /** A Builder can build a RowTransformer. */ - public static class Builder { - private final List projections = new ArrayList<>(); - private final List sourceTypes = new ArrayList<>(); - - public RowTransformer build() { - return new RowTransformer(sourceTypes, projections); - } - - public Builder addProjection(Projection projection) { - this.projections.add(projection); - return this; - } - - public Builder addProjections(Projection... projections) { - this.projections.addAll(Arrays.asList(projections)); - return this; - } - - public Builder addSourceFieldType(DataType fieldType) { - this.sourceTypes.add(fieldType); - return this; - } - - public Builder addSourceFieldTypes(DataType... fieldTypes) { - this.sourceTypes.addAll(Arrays.asList(fieldTypes)); - return this; - } - - public Builder addSourceFieldTypes(List fieldTypes) { - this.sourceTypes.addAll(fieldTypes); - return this; - } - } - - private final List projections; - - private final List sourceFieldTypes; - - private RowTransformer(List sourceTypes, List projections) { - this.sourceFieldTypes = ImmutableList.copyOf(requireNonNull(sourceTypes)); - this.projections = ImmutableList.copyOf(requireNonNull(projections)); - } - - /** - * Transforms input row to a output row according projections operator passed on creation of this - * RowTransformer. - * - * @param inRow input row that need to be transformed. - * @return a row that is already transformed. - */ - public Row transform(Row inRow) { - // After transform the length of row is probably not same as the input row. - // we need calculate the new length. - Row outRow = ObjectRowImpl.create(newRowLength()); - - int offset = 0; - for (int i = 0; i < inRow.fieldCount(); i++) { - Object inVal = inRow.get(i, sourceFieldTypes.get(i)); - Projection p = getProjection(i); - p.set(inVal, outRow, offset); - offset += p.size(); - } - return outRow; - } - - private Projection getProjection(int index) { - return projections.get(index); - } - - /** - * Collect output row's length. - * - * @return a int which is the new length of output row. - */ - private int newRowLength() { - return this.projections.stream().reduce(0, (sum, p) -> sum += p.size(), (s1, s2) -> s1 + s2); - } - - public List getTypes() { - return projections - .stream() - .flatMap(proj -> proj.getTypes().stream()) - .collect(Collectors.toList()); - } -} diff --git a/src/main/java/org/tikv/operation/transformer/Skip.java b/src/main/java/org/tikv/operation/transformer/Skip.java deleted file mode 100644 index 9960889d43..0000000000 --- a/src/main/java/org/tikv/operation/transformer/Skip.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.operation.transformer; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import org.tikv.row.Row; -import org.tikv.types.DataType; - -public class Skip implements Projection { - public static final Skip SKIP_OP = new Skip(); - - @Override - public void set(Object value, Row row, int pos) {} - - @Override - public int size() { - return 0; - } - - @Override - public List getTypes() { - return ImmutableList.of(); - } -} diff --git a/src/main/java/org/tikv/predicates/IndexRange.java b/src/main/java/org/tikv/predicates/IndexRange.java deleted file mode 100644 index d946a5bd99..0000000000 --- a/src/main/java/org/tikv/predicates/IndexRange.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.predicates; - -import com.google.common.collect.Range; -import org.tikv.key.Key; -import org.tikv.key.TypedKey; - -public class IndexRange { - private Key accessKey; - private Range range; - - public IndexRange(Key accessKey, Range range) { - this.accessKey = accessKey; - this.range = range; - } - - public Key getAccessKey() { - return accessKey; - } - - public boolean hasAccessKey() { - return accessKey != null; - } - - public boolean hasRange() { - return range != null; - } - - public Range getRange() { - return range; - } -} diff --git a/src/main/java/org/tikv/predicates/PredicateUtils.java b/src/main/java/org/tikv/predicates/PredicateUtils.java deleted file mode 100644 index 91b8d106a7..0000000000 --- a/src/main/java/org/tikv/predicates/PredicateUtils.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.predicates; - -import static java.util.Objects.requireNonNull; -import static org.tikv.expression.LogicalBinaryExpression.and; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Range; -import java.util.*; -import org.tikv.exception.TiExpressionException; -import org.tikv.expression.ColumnRef; -import org.tikv.expression.Expression; -import org.tikv.expression.Visitor; -import org.tikv.expression.visitor.DefaultVisitor; -import org.tikv.expression.visitor.IndexRangeBuilder; -import org.tikv.key.CompoundKey; -import org.tikv.key.Key; -import org.tikv.key.TypedKey; -import org.tikv.meta.TiIndexInfo; -import org.tikv.meta.TiTableInfo; - -public class PredicateUtils { - public static Expression mergeCNFExpressions(List exprs) { - requireNonNull(exprs, "Expression list is null"); - if (exprs.size() == 0) return null; - if (exprs.size() == 1) return exprs.get(0); - - return and(exprs.get(0), mergeCNFExpressions(exprs.subList(1, exprs.size()))); - } - - public static Set extractColumnRefFromExpression(Expression expr) { - Set columnRefs = new HashSet<>(); - Visitor> visitor = - new DefaultVisitor>() { - @Override - protected Void visit(ColumnRef node, Set context) { - context.add(node); - return null; - } - }; - - expr.accept(visitor, columnRefs); - return columnRefs; - } - - /** - * Build index ranges from access points and access conditions - * - * @param pointPredicates conditions converting to a single point access - * @param rangePredicate conditions converting to a range - * @return Index Range for scan - */ - public static List expressionToIndexRanges( - List pointPredicates, - Optional rangePredicate, - TiTableInfo table, - TiIndexInfo index) { - requireNonNull(pointPredicates, "pointPredicates is null"); - requireNonNull(rangePredicate, "rangePredicate is null"); - ImmutableList.Builder builder = ImmutableList.builder(); - IndexRangeBuilder indexRangeBuilder = new IndexRangeBuilder(table, index); - - if (pointPredicates.size() != 0) { - List pointKeys = expressionToPoints(pointPredicates, table, index); - for (Key key : pointKeys) { - if (rangePredicate.isPresent()) { - Set> ranges = indexRangeBuilder.buildRange(rangePredicate.get()); - for (Range range : ranges) { - builder.add(new IndexRange(key, range)); - } - } else { - // no predicates with point keys leads to empty range encoding - builder.add(new IndexRange(key, null)); - } - } - } else { - if (rangePredicate.isPresent()) { - Set> ranges = indexRangeBuilder.buildRange(rangePredicate.get()); - for (Range range : ranges) { - builder.add(new IndexRange(null, range)); - } - } else { - // no filter at all means full range - builder.add(new IndexRange(null, Range.all())); - } - } - - return builder.build(); - } - - /** - * Turn access conditions into list of points Each condition is bound to single key We pick up - * single condition for each index key and disregard if multiple EQ conditions in DNF - * - * @param pointPredicates expressions that convertible to access points - * @return access points for each index - */ - private static List expressionToPoints( - List pointPredicates, TiTableInfo table, TiIndexInfo index) { - requireNonNull(pointPredicates, "pointPredicates cannot be null"); - - List resultKeys = new ArrayList<>(); - IndexRangeBuilder indexRangeBuilder = new IndexRangeBuilder(table, index); - - for (int i = 0; i < pointPredicates.size(); i++) { - Expression predicate = pointPredicates.get(i); - try { - // each expr will be expand to one or more points - Set> ranges = indexRangeBuilder.buildRange(predicate); - List points = rangesToPoint(ranges); - resultKeys = joinKeys(resultKeys, points); - } catch (Exception e) { - throw new TiExpressionException( - String.format("Error converting access points %s", predicate), e); - } - } - return resultKeys; - } - - // Convert ranges of equal condition points to List of TypedKeys - private static List rangesToPoint(Set> ranges) { - requireNonNull(ranges, "ranges is null"); - ImmutableList.Builder builder = ImmutableList.builder(); - for (Range range : ranges) { - // test if range is a point - if (range.hasLowerBound() - && range.hasUpperBound() - && range.lowerEndpoint().equals(range.upperEndpoint())) { - builder.add(range.lowerEndpoint()); - } else { - throw new TiExpressionException("Cannot convert range to point"); - } - } - return builder.build(); - } - - private static List joinKeys(List lhsKeys, List rhsKeys) { - requireNonNull(lhsKeys, "lhsKeys is null"); - requireNonNull(rhsKeys, "rhsKeys is null"); - if (lhsKeys.isEmpty()) { - return rhsKeys; - } - if (rhsKeys.isEmpty()) { - return lhsKeys; - } - ImmutableList.Builder builder = ImmutableList.builder(); - for (Key lKey : lhsKeys) { - for (Key rKey : rhsKeys) { - builder.add(CompoundKey.concat(lKey, rKey)); - } - } - return builder.build(); - } -} diff --git a/src/main/java/org/tikv/predicates/ScanAnalyzer.java b/src/main/java/org/tikv/predicates/ScanAnalyzer.java deleted file mode 100644 index 125decea8a..0000000000 --- a/src/main/java/org/tikv/predicates/ScanAnalyzer.java +++ /dev/null @@ -1,402 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.predicates; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; -import static org.tikv.predicates.PredicateUtils.expressionToIndexRanges; -import static org.tikv.util.KeyRangeUtils.makeCoprocRange; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.BoundType; -import com.google.common.collect.Range; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import org.tikv.exception.TiClientInternalException; -import org.tikv.expression.Expression; -import org.tikv.expression.visitor.IndexMatcher; -import org.tikv.expression.visitor.MetaResolver; -import org.tikv.key.IndexKey; -import org.tikv.key.Key; -import org.tikv.key.RowKey; -import org.tikv.key.TypedKey; -import org.tikv.kvproto.Coprocessor.KeyRange; -import org.tikv.meta.TiColumnInfo; -import org.tikv.meta.TiIndexColumn; -import org.tikv.meta.TiIndexInfo; -import org.tikv.meta.TiTableInfo; -import org.tikv.statistics.IndexStatistics; -import org.tikv.statistics.TableStatistics; -import org.tikv.types.DataType; - -public class ScanAnalyzer { - private static final double INDEX_SCAN_COST_FACTOR = 1.2; - private static final double TABLE_SCAN_COST_FACTOR = 1.0; - private static final double DOUBLE_READ_COST_FACTOR = TABLE_SCAN_COST_FACTOR * 3; - - public static class ScanPlan { - ScanPlan( - List keyRanges, - Set filters, - TiIndexInfo index, - double cost, - boolean isDoubleRead, - double estimatedRowCount) { - this.filters = filters; - this.keyRanges = keyRanges; - this.cost = cost; - this.index = index; - this.isDoubleRead = isDoubleRead; - this.estimatedRowCount = estimatedRowCount; - } - - private final List keyRanges; - private final Set filters; - private final double cost; - private TiIndexInfo index; - private final boolean isDoubleRead; - private final double estimatedRowCount; - - public double getEstimatedRowCount() { - return estimatedRowCount; - } - - public List getKeyRanges() { - return keyRanges; - } - - public Set getFilters() { - return filters; - } - - public double getCost() { - return cost; - } - - public boolean isIndexScan() { - return index != null && !index.isFakePrimaryKey(); - } - - public TiIndexInfo getIndex() { - return index; - } - - public boolean isDoubleRead() { - return isDoubleRead; - } - } - - public ScanPlan buildScan( - List columnList, List conditions, TiTableInfo table) { - return buildScan(columnList, conditions, table, null); - } - - // Build scan plan picking access path with lowest cost by estimation - public ScanPlan buildScan( - List columnList, - List conditions, - TiTableInfo table, - TableStatistics tableStatistics) { - ScanPlan minPlan = buildTableScan(conditions, table, tableStatistics); - double minCost = minPlan.getCost(); - for (TiIndexInfo index : table.getIndices()) { - ScanPlan plan = buildScan(columnList, conditions, index, table, tableStatistics); - if (plan.getCost() < minCost) { - minPlan = plan; - minCost = plan.getCost(); - } - } - return minPlan; - } - - public ScanPlan buildTableScan( - List conditions, TiTableInfo table, TableStatistics tableStatistics) { - TiIndexInfo pkIndex = TiIndexInfo.generateFakePrimaryKeyIndex(table); - return buildScan(table.getColumns(), conditions, pkIndex, table, tableStatistics); - } - - public ScanPlan buildScan( - List columnList, - List conditions, - TiIndexInfo index, - TiTableInfo table, - TableStatistics tableStatistics) { - requireNonNull(table, "Table cannot be null to encoding keyRange"); - requireNonNull(conditions, "conditions cannot be null to encoding keyRange"); - - MetaResolver.resolve(conditions, table); - - ScanSpec result = extractConditions(conditions, table, index); - - double cost = SelectivityCalculator.calcPseudoSelectivity(result); - - List irs = - expressionToIndexRanges( - result.getPointPredicates(), result.getRangePredicate(), table, index); - - List keyRanges; - boolean isDoubleRead = false; - double estimatedRowCount = -1; - // table name and columns - int tableSize = table.getColumns().size() + 1; - - if (index == null || index.isFakePrimaryKey()) { - if (tableStatistics != null) { - cost = 100.0; // Full table scan cost - // TODO: Fine-grained statistics usage - } - keyRanges = buildTableScanKeyRange(table, irs); - cost *= tableSize * TABLE_SCAN_COST_FACTOR; - } else { - if (tableStatistics != null) { - long totalRowCount = tableStatistics.getCount(); - IndexStatistics indexStatistics = tableStatistics.getIndexHistMap().get(index.getId()); - if (conditions.isEmpty()) { - cost = 100.0; // Full index scan cost - // TODO: Fine-grained statistics usage - estimatedRowCount = totalRowCount; - } else if (indexStatistics != null) { - double idxRangeRowCnt = indexStatistics.getRowCount(irs); - // guess the percentage of rows hit - cost = 100.0 * idxRangeRowCnt / totalRowCount; - estimatedRowCount = idxRangeRowCnt; - } - } - isDoubleRead = !isCoveringIndex(columnList, index, table.isPkHandle()); - // table name, index and handle column - int indexSize = index.getIndexColumns().size() + 2; - if (isDoubleRead) { - cost *= tableSize * DOUBLE_READ_COST_FACTOR + indexSize * INDEX_SCAN_COST_FACTOR; - } else { - cost *= indexSize * INDEX_SCAN_COST_FACTOR; - } - keyRanges = buildIndexScanKeyRange(table, index, irs); - } - - return new ScanPlan( - keyRanges, result.getResidualPredicates(), index, cost, isDoubleRead, estimatedRowCount); - } - - @VisibleForTesting - List buildTableScanKeyRange(TiTableInfo table, List indexRanges) { - requireNonNull(table, "Table is null"); - requireNonNull(indexRanges, "indexRanges is null"); - - List ranges = new ArrayList<>(indexRanges.size()); - for (IndexRange ir : indexRanges) { - Key startKey; - Key endKey; - if (ir.hasAccessKey()) { - checkArgument( - !ir.hasRange(), "Table scan must have one and only one access condition / point"); - - Key key = ir.getAccessKey(); - checkArgument(key instanceof TypedKey, "Table scan key range must be typed key"); - TypedKey typedKey = (TypedKey) key; - startKey = RowKey.toRowKey(table.getId(), typedKey); - endKey = startKey.next(); - } else if (ir.hasRange()) { - checkArgument( - !ir.hasAccessKey(), "Table scan must have one and only one access condition / point"); - Range r = ir.getRange(); - - if (!r.hasLowerBound()) { - // -INF - startKey = RowKey.createMin(table.getId()); - } else { - // Comparision with null should be filtered since it yields unknown always - startKey = RowKey.toRowKey(table.getId(), r.lowerEndpoint()); - if (r.lowerBoundType().equals(BoundType.OPEN)) { - startKey = startKey.next(); - } - } - - if (!r.hasUpperBound()) { - // INF - endKey = RowKey.createBeyondMax(table.getId()); - } else { - endKey = RowKey.toRowKey(table.getId(), r.upperEndpoint()); - if (r.upperBoundType().equals(BoundType.CLOSED)) { - endKey = endKey.next(); - } - } - } else { - throw new TiClientInternalException("Empty access conditions"); - } - - // This range only possible when < MIN or > MAX - if (!startKey.equals(endKey)) { - ranges.add(makeCoprocRange(startKey.toByteString(), endKey.toByteString())); - } - } - - return ranges; - } - - @VisibleForTesting - List buildIndexScanKeyRange( - TiTableInfo table, TiIndexInfo index, List indexRanges) { - requireNonNull(table, "Table cannot be null to encoding keyRange"); - requireNonNull(index, "Index cannot be null to encoding keyRange"); - requireNonNull(indexRanges, "indexRanges cannot be null to encoding keyRange"); - - List ranges = new ArrayList<>(indexRanges.size()); - - for (IndexRange ir : indexRanges) { - Key pointKey = ir.hasAccessKey() ? ir.getAccessKey() : Key.EMPTY; - - Range range = ir.getRange(); - Key lPointKey; - Key uPointKey; - - Key lKey; - Key uKey; - if (!ir.hasRange()) { - lPointKey = pointKey; - uPointKey = pointKey.next(); - - lKey = Key.EMPTY; - uKey = Key.EMPTY; - } else { - lPointKey = pointKey; - uPointKey = pointKey; - - if (!range.hasLowerBound()) { - // -INF - lKey = Key.NULL; - } else { - lKey = range.lowerEndpoint(); - if (range.lowerBoundType().equals(BoundType.OPEN)) { - lKey = lKey.next(); - } - } - - if (!range.hasUpperBound()) { - // INF - uKey = Key.MAX; - } else { - uKey = range.upperEndpoint(); - if (range.upperBoundType().equals(BoundType.CLOSED)) { - uKey = uKey.next(); - } - } - } - IndexKey lbsKey = IndexKey.toIndexKey(table.getId(), index.getId(), lPointKey, lKey); - IndexKey ubsKey = IndexKey.toIndexKey(table.getId(), index.getId(), uPointKey, uKey); - - ranges.add(makeCoprocRange(lbsKey.toByteString(), ubsKey.toByteString())); - } - - return ranges; - } - - boolean isCoveringIndex( - List columns, TiIndexInfo indexColumns, boolean pkIsHandle) { - for (TiColumnInfo colInfo : columns) { - if (pkIsHandle && colInfo.isPrimaryKey()) { - continue; - } - if (colInfo.getId() == -1) { - continue; - } - boolean isIndexColumn = false; - for (TiIndexColumn indexCol : indexColumns.getIndexColumns()) { - boolean isFullLength = - indexCol.getLength() == DataType.UNSPECIFIED_LEN - || indexCol.getLength() == colInfo.getType().getLength(); - if (colInfo.getName().equalsIgnoreCase(indexCol.getName()) && isFullLength) { - isIndexColumn = true; - break; - } - } - if (!isIndexColumn) { - return false; - } - } - return true; - } - - @VisibleForTesting - public static ScanSpec extractConditions( - List conditions, TiTableInfo table, TiIndexInfo index) { - // 0. Different than TiDB implementation, here logic has been unified for TableScan and - // IndexScan by - // adding fake index on clustered table's pk - // 1. Generate access point based on equal conditions - // 2. Cut access point condition if index is not continuous - // 3. Push back prefix index conditions since prefix index retrieve more result than needed - // 4. For remaining indexes (since access conditions consume some index, and they will - // not be used in filter push down later), find continuous matching index until first unmatched - // 5. Push back index related filter if prefix index, for remaining filters - // Equal conditions needs to be process first according to index sequence - // When index is null, no access condition can be applied - ScanSpec.Builder specBuilder = new ScanSpec.Builder(table, index); - if (index != null) { - Set visited = new HashSet<>(); - IndexMatchingLoop: - for (int i = 0; i < index.getIndexColumns().size(); i++) { - // for each index column try matches an equal condition - // and push remaining back - // TODO: if more than one equal conditions match an - // index, it likely yields nothing. Maybe a check needed - // to simplify it to a false condition - TiIndexColumn col = index.getIndexColumns().get(i); - IndexMatcher eqMatcher = IndexMatcher.equalOnlyMatcher(col); - boolean found = false; - // For first prefix index encountered, it equals to a range - // and we cannot push equal conditions further - for (Expression cond : conditions) { - if (visited.contains(cond)) { - continue; - } - if (eqMatcher.match(cond)) { - specBuilder.addPointPredicate(col, cond); - if (col.isPrefixIndex()) { - specBuilder.addResidualPredicate(cond); - break IndexMatchingLoop; - } - visited.add(cond); - found = true; - break; - } - } - if (!found) { - // For first "broken index chain piece" - // search for a matching range condition - IndexMatcher matcher = IndexMatcher.matcher(col); - for (Expression cond : conditions) { - if (visited.contains(cond)) { - continue; - } - if (matcher.match(cond)) { - specBuilder.addRangePredicate(col, cond); - if (col.isPrefixIndex()) { - specBuilder.addResidualPredicate(cond); - break; - } - } - } - break; - } - } - } - - specBuilder.addAllPredicates(conditions); - return specBuilder.build(); - } -} diff --git a/src/main/java/org/tikv/predicates/ScanSpec.java b/src/main/java/org/tikv/predicates/ScanSpec.java deleted file mode 100644 index 00ce7d0cf5..0000000000 --- a/src/main/java/org/tikv/predicates/ScanSpec.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.predicates; - -import static java.util.Objects.requireNonNull; -import static org.tikv.predicates.PredicateUtils.mergeCNFExpressions; - -import com.google.common.collect.ImmutableList; -import java.util.*; -import org.tikv.exception.TiExpressionException; -import org.tikv.expression.Expression; -import org.tikv.meta.TiColumnInfo; -import org.tikv.meta.TiIndexColumn; -import org.tikv.meta.TiIndexInfo; -import org.tikv.meta.TiTableInfo; -import org.tikv.types.DataType; - -public class ScanSpec { - public static class Builder { - private final IdentityHashMap> pointPredicates = - new IdentityHashMap<>(); - private TiIndexColumn rangeColumn; - private final TiTableInfo table; - private final TiIndexInfo index; - private final List rangePredicates = new ArrayList<>(); - private final List residualPredicates = new ArrayList<>(); - private Set residualCandidates = new HashSet<>(); - - public Builder(TiTableInfo table, TiIndexInfo index) { - this.table = table; - this.index = index; - } - - public void addResidualPredicate(Expression predicate) { - residualPredicates.add(predicate); - } - - public void addAllPredicates(List predicates) { - residualCandidates.addAll(predicates); - } - - public void addPointPredicate(TiIndexColumn col, Expression predicate) { - requireNonNull(col, "index column is null"); - requireNonNull(predicate, "predicate is null"); - if (pointPredicates.containsKey(col)) { - List predicates = pointPredicates.get(col); - predicates.add(predicate); - } else { - List predicates = new ArrayList<>(); - predicates.add(predicate); - pointPredicates.put(col, predicates); - } - } - - public void addRangePredicate(TiIndexColumn col, Expression predicate) { - requireNonNull(col, "col is null"); - if (rangeColumn == null) { - rangeColumn = col; - } else if (!rangeColumn.equals(col)) { - throw new TiExpressionException("Cannot reset range predicates"); - } - rangePredicates.add(predicate); - } - - public ScanSpec build() { - List points = new ArrayList<>(); - List pointTypes = new ArrayList<>(); - Set pushedPredicates = new HashSet<>(); - if (index != null) { - for (TiIndexColumn indexColumn : index.getIndexColumns()) { - List predicates = pointPredicates.get(indexColumn); - if (predicates == null) { - break; - } - pushedPredicates.addAll(predicates); - TiColumnInfo tiColumnInfo = table.getColumn(indexColumn.getOffset()); - DataType type = tiColumnInfo.getType(); - points.add(mergeCNFExpressions(predicates)); - pointTypes.add(type); - } - } - Optional newRangePred = - rangePredicates.isEmpty() - ? Optional.empty() - : Optional.of(mergeCNFExpressions(rangePredicates)); - pushedPredicates.addAll(rangePredicates); - - Set newResidualPredicates = new HashSet<>(residualPredicates); - for (Expression pred : residualCandidates) { - if (!pushedPredicates.contains(pred)) { - newResidualPredicates.add(pred); - } - } - - Optional rangeType; - if (rangeColumn == null) { - rangeType = Optional.empty(); - } else { - TiColumnInfo col = table.getColumn(rangeColumn.getOffset()); - rangeType = Optional.of(col.getType()); - } - - return new ScanSpec(ImmutableList.copyOf(points), newRangePred, newResidualPredicates); - } - } - - private final List pointPredicates; - private final Optional rangePredicate; - private final Set residualPredicates; - - private ScanSpec( - List pointPredicates, - Optional rangePredicate, - Set residualPredicates) { - this.pointPredicates = pointPredicates; - this.rangePredicate = rangePredicate; - this.residualPredicates = residualPredicates; - } - - public List getPointPredicates() { - return pointPredicates; - } - - public Optional getRangePredicate() { - return rangePredicate; - } - - public Set getResidualPredicates() { - return residualPredicates; - } -} diff --git a/src/main/java/org/tikv/predicates/SelectivityCalculator.java b/src/main/java/org/tikv/predicates/SelectivityCalculator.java deleted file mode 100644 index 076d93bbf0..0000000000 --- a/src/main/java/org/tikv/predicates/SelectivityCalculator.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.predicates; - -import java.util.Optional; -import org.tikv.expression.Expression; -import org.tikv.expression.visitor.DefaultVisitor; -import org.tikv.expression.visitor.PseudoCostCalculator; - -public class SelectivityCalculator extends DefaultVisitor { - public static double calcPseudoSelectivity(ScanSpec spec) { - Optional rangePred = spec.getRangePredicate(); - double cost = 100.0; - if (spec.getPointPredicates() != null) { - for (Expression expr : spec.getPointPredicates()) { - cost *= PseudoCostCalculator.calculateCost(expr); - } - } - if (rangePred.isPresent()) { - cost *= PseudoCostCalculator.calculateCost(rangePred.get()); - } - return cost; - } - - @Override - protected Double process(Expression node, Void context) { - return 1.0; - } -} diff --git a/src/main/java/org/tikv/region/RegionManager.java b/src/main/java/org/tikv/region/RegionManager.java index 0e670b07e4..441804509a 100644 --- a/src/main/java/org/tikv/region/RegionManager.java +++ b/src/main/java/org/tikv/region/RegionManager.java @@ -177,6 +177,9 @@ public class RegionManager { if (store.getState().equals(StoreState.Tombstone)) { return null; } + if (logger.isDebugEnabled()) { + logger.debug(String.format("getStoreById ID[%s] -> Store[%s]", id, store)); + } storeCache.put(id, store); return store; } catch (Exception e) { diff --git a/src/main/java/org/tikv/region/RegionStoreClient.java b/src/main/java/org/tikv/region/RegionStoreClient.java index dee649b10a..c215aaa2cd 100644 --- a/src/main/java/org/tikv/region/RegionStoreClient.java +++ b/src/main/java/org/tikv/region/RegionStoreClient.java @@ -19,26 +19,17 @@ package org.tikv.region; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static org.tikv.region.RegionStoreClient.RequestTypes.REQ_TYPE_DAG; import static org.tikv.util.BackOffFunction.BackOffFuncType.BoRegionMiss; import static org.tikv.util.BackOffFunction.BackOffFuncType.BoTxnLockFast; import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; -import com.pingcap.tidb.tipb.DAGRequest; -import com.pingcap.tidb.tipb.SelectResponse; import io.grpc.ManagedChannel; import java.util.*; import java.util.function.Supplier; -import jline.internal.TestAccessible; -import org.apache.hadoop.hive.ql.lockmgr.LockException; import org.apache.log4j.Logger; import org.tikv.AbstractGRPCClient; import org.tikv.TiSession; import org.tikv.exception.*; -import org.tikv.kvproto.Coprocessor; -import org.tikv.kvproto.Coprocessor.KeyRange; -import org.tikv.kvproto.Errorpb; import org.tikv.kvproto.Kvrpcpb.BatchGetRequest; import org.tikv.kvproto.Kvrpcpb.BatchGetResponse; import org.tikv.kvproto.Kvrpcpb.Context; @@ -60,34 +51,13 @@ import org.tikv.kvproto.TikvGrpc; import org.tikv.kvproto.TikvGrpc.TikvBlockingStub; import org.tikv.kvproto.TikvGrpc.TikvStub; import org.tikv.operation.KVErrorHandler; -import org.tikv.streaming.StreamingResponse; import org.tikv.txn.Lock; import org.tikv.txn.LockResolverClient; -import org.tikv.util.BackOffFunction; import org.tikv.util.BackOffer; -import org.tikv.util.ConcreteBackOffer; -import org.tikv.util.RangeSplitter; // RegionStore itself is not thread-safe public class RegionStoreClient extends AbstractGRPCClient implements RegionErrorReceiver { - public enum RequestTypes { - REQ_TYPE_SELECT(101), - REQ_TYPE_INDEX(102), - REQ_TYPE_DAG(103), - REQ_TYPE_ANALYZE(104), - BATCH_ROW_COUNT(64); - - private final int value; - - RequestTypes(int value) { - this.value = value; - } - - public int getValue() { - return value; - } - } private static final Logger logger = Logger.getLogger(RegionStoreClient.class); private TiRegion region; @@ -99,7 +69,7 @@ public class RegionStoreClient extends AbstractGRPCClient coprocess( - BackOffer backOffer, - DAGRequest req, - List ranges, - Queue responseQueue) { - if (req == null || ranges == null || req.getExecutorsCount() < 1) { - throw new IllegalArgumentException("Invalid coprocess argument!"); - } - - Supplier reqToSend = - () -> - Coprocessor.Request.newBuilder() - .setContext(region.getContext()) - .setTp(REQ_TYPE_DAG.getValue()) - .setData(req.toByteString()) - .addAllRanges(ranges) - .build(); - - // we should handle the region error ourselves - KVErrorHandler handler = - new KVErrorHandler<>( - regionManager, - this, - region, - resp -> resp.hasRegionError() ? resp.getRegionError() : null); - Coprocessor.Response resp = - callWithRetry(backOffer, TikvGrpc.METHOD_COPROCESSOR, reqToSend, handler); - return handleCopResponse(backOffer, resp, ranges, responseQueue); - } - - // handleCopResponse checks coprocessor Response for region split and lock, - // returns more tasks when that happens, or handles the response if no error. - // if we're handling streaming coprocessor response, lastRange is the range of last - // successful response, otherwise it's nil. - private List handleCopResponse( - BackOffer backOffer, - Coprocessor.Response response, - List ranges, - Queue responseQueue) { - if (response.hasRegionError()) { - Errorpb.Error regionError = response.getRegionError(); - backOffer.doBackOff( - BackOffFunction.BackOffFuncType.BoRegionMiss, new GrpcException(regionError.toString())); - logger.warn("Re-splitting region task due to region error:" + regionError.getMessage()); - // Split ranges - return RangeSplitter.newSplitter(session.getRegionManager()).splitRangeByRegion(ranges); - } - - if (response.hasLocked()) { - logger.debug(String.format("coprocessor encounters locks: %s", response.getLocked())); - Lock lock = new Lock(response.getLocked()); - boolean ok = lockResolverClient.resolveLocks(backOffer, new ArrayList<>(Arrays.asList(lock))); - if (!ok) { - backOffer.doBackOff(BoTxnLockFast, new LockException()); - } - // Split ranges - return RangeSplitter.newSplitter(session.getRegionManager()).splitRangeByRegion(ranges); - } - - String otherError = response.getOtherError(); - if (otherError != null && !otherError.isEmpty()) { - logger.warn(String.format("Other error occurred, message: %s", otherError)); - throw new GrpcException(otherError); - } - - responseQueue.offer(coprocessorHelper(response)); - return null; - } - - // TODO: wait for future fix - // coprocessStreaming doesn't handle split error - // future work should handle it and do the resolve - // locks correspondingly - public Iterator coprocessStreaming(DAGRequest req, List ranges) { - Supplier reqToSend = - () -> - Coprocessor.Request.newBuilder() - .setContext(region.getContext()) - // TODO: If no executors...? - .setTp(REQ_TYPE_DAG.getValue()) - .setData(req.toByteString()) - .addAllRanges(ranges) - .build(); - - KVErrorHandler handler = - new KVErrorHandler<>( - regionManager, - this, - region, - StreamingResponse::getFirstRegionError // TODO: handle all errors in streaming respinse - ); - - StreamingResponse responseIterator = - this.callServerStreamingWithRetry( - ConcreteBackOffer.newCopNextMaxBackOff(), - TikvGrpc.METHOD_COPROCESSOR_STREAM, - reqToSend, - handler); - return coprocessorHelper(responseIterator); - } - - private Iterator coprocessorHelper(StreamingResponse response) { - Iterator responseIterator = response.iterator(); - // If we got nothing to handle, return null - if (!responseIterator.hasNext()) return null; - - // Simply wrap it - return new Iterator() { - @Override - public boolean hasNext() { - return responseIterator.hasNext(); - } - - @Override - public SelectResponse next() { - return coprocessorHelper(responseIterator.next()); - } - }; - } - - private SelectResponse coprocessorHelper(Coprocessor.Response resp) { - try { - SelectResponse selectResp = SelectResponse.parseFrom(resp.getData()); - if (selectResp.hasError()) { - throw new SelectException(selectResp.getError(), selectResp.getError().getMsg()); - } - return selectResp; - } catch (InvalidProtocolBufferException e) { - throw new TiClientInternalException("Error parsing protobuf for coprocessor response.", e); - } - } - public TiSession getSession() { return session; } diff --git a/src/main/java/org/tikv/row/DefaultRowReader.java b/src/main/java/org/tikv/row/DefaultRowReader.java deleted file mode 100644 index f1f5880a1e..0000000000 --- a/src/main/java/org/tikv/row/DefaultRowReader.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.row; - -import org.tikv.codec.CodecDataInput; -import org.tikv.types.DataType; - -public class DefaultRowReader implements RowReader { - private final CodecDataInput cdi; - - public static DefaultRowReader create(CodecDataInput cdi) { - return new DefaultRowReader(cdi); - } - - DefaultRowReader(CodecDataInput cdi) { - this.cdi = cdi; - } - - public Row readRow(DataType[] dataTypes) { - int length = dataTypes.length; - Row row = ObjectRowImpl.create(length); - for (int i = 0; i < length; i++) { - DataType type = dataTypes[i]; - if (type.isNextNull(cdi)) { - cdi.readUnsignedByte(); - row.setNull(i); - } else { - row.set(i, type, type.decode(cdi)); - } - } - return row; - } -} diff --git a/src/main/java/org/tikv/row/ObjectRowImpl.java b/src/main/java/org/tikv/row/ObjectRowImpl.java deleted file mode 100644 index 30bc9434e3..0000000000 --- a/src/main/java/org/tikv/row/ObjectRowImpl.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.row; - -import java.math.BigDecimal; -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; -import org.tikv.types.DataType; - -// A dummy implementation of Row interface -// Using non-memory compact format -public class ObjectRowImpl implements Row { - private final Object[] values; - - public static Row create(int fieldCount) { - return new ObjectRowImpl(fieldCount); - } - - private ObjectRowImpl(int fieldCount) { - values = new Object[fieldCount]; - } - - @Override - public void setNull(int pos) { - values[pos] = null; - } - - @Override - public boolean isNull(int pos) { - return values[pos] == null; - } - - @Override - public void setFloat(int pos, float v) { - values[pos] = v; - } - - @Override - public float getFloat(int pos) { - return (float) values[pos]; - } - - @Override - public void setInteger(int pos, int v) { - values[pos] = v; - } - - @Override - public int getInteger(int pos) { - return (int) values[pos]; - } - - @Override - public void setShort(int pos, short v) { - values[pos] = v; - } - - @Override - public short getShort(int pos) { - return (short) values[pos]; - } - - @Override - public void setDouble(int pos, double v) { - values[pos] = v; - } - - @Override - public double getDouble(int pos) { - // Null should be handled by client code with isNull - // below all get method behave the same - return (double) values[pos]; - } - - @Override - public void setLong(int pos, long v) { - values[pos] = v; - } - - @Override - public long getLong(int pos) { - // Null should be handled by client code with isNull - // below all get method behave the same - return (long) values[pos]; - } - - @Override - public long getUnsignedLong(int pos) { - return ((BigDecimal) values[pos]).longValue(); - } - - @Override - public void setString(int pos, String v) { - values[pos] = v; - } - - @Override - public String getString(int pos) { - return (String) values[pos]; - } - - @Override - public void setTime(int pos, Time v) { - values[pos] = v; - } - - @Override - public Date getTime(int pos) { - return (Date) values[pos]; - } - - @Override - public void setTimestamp(int pos, Timestamp v) { - values[pos] = v; - } - - @Override - public Timestamp getTimestamp(int pos) { - return (Timestamp) values[pos]; - } - - @Override - public void setDate(int pos, Date v) { - values[pos] = v; - } - - @Override - public Date getDate(int pos) { - return (Date) values[pos]; - } - - @Override - public void setBytes(int pos, byte[] v) { - values[pos] = v; - } - - @Override - public byte[] getBytes(int pos) { - return (byte[]) values[pos]; - } - - @Override - public void set(int pos, DataType type, Object v) { - // Ignore type for this implementation since no serialization happens - values[pos] = v; - } - - @Override - public Object get(int pos, DataType type) { - // Ignore type for this implementation since no serialization happens - return values[pos]; - } - - @Override - public int fieldCount() { - return values.length; - } -} diff --git a/src/main/java/org/tikv/row/Row.java b/src/main/java/org/tikv/row/Row.java deleted file mode 100644 index 5a4c1c9330..0000000000 --- a/src/main/java/org/tikv/row/Row.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.row; - -import java.sql.Date; -import java.sql.Time; -import java.sql.Timestamp; -import org.tikv.types.DataType; - -/** - * Even in case of mem-buffer-based row we can ignore field types when en/decoding if we put some - * padding bits for fixed length and use fixed length index for var-length - */ -public interface Row { - void setNull(int pos); - - boolean isNull(int pos); - - void setFloat(int pos, float v); - - float getFloat(int pos); - - void setDouble(int pos, double v); - - double getDouble(int pos); - - void setInteger(int pos, int v); - - int getInteger(int pos); - - void setShort(int pos, short v); - - short getShort(int pos); - - void setLong(int pos, long v); - - long getLong(int pos); - - long getUnsignedLong(int pos); - - void setString(int pos, String v); - - String getString(int pos); - - void setTime(int pos, Time v); - - Date getTime(int pos); - - void setTimestamp(int pos, Timestamp v); - - Timestamp getTimestamp(int pos); - - void setDate(int pos, Date v); - - Date getDate(int pos); - - void setBytes(int pos, byte[] v); - - byte[] getBytes(int pos); - - void set(int pos, DataType type, Object v); - - Object get(int pos, DataType type); - - int fieldCount(); -} diff --git a/src/main/java/org/tikv/row/RowReader.java b/src/main/java/org/tikv/row/RowReader.java deleted file mode 100644 index d417e080e5..0000000000 --- a/src/main/java/org/tikv/row/RowReader.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.row; - -import org.tikv.types.DataType; - -public interface RowReader { - Row readRow(DataType[] dataTypes); -} diff --git a/src/main/java/org/tikv/row/RowReaderFactory.java b/src/main/java/org/tikv/row/RowReaderFactory.java deleted file mode 100644 index af77b89865..0000000000 --- a/src/main/java/org/tikv/row/RowReaderFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.row; - -import org.tikv.codec.CodecDataInput; - -public class RowReaderFactory { - public static RowReader createRowReader(CodecDataInput cdi) { - return new DefaultRowReader(cdi); - } -} diff --git a/src/main/java/org/tikv/statistics/Bucket.java b/src/main/java/org/tikv/statistics/Bucket.java deleted file mode 100644 index 9b72448010..0000000000 --- a/src/main/java/org/tikv/statistics/Bucket.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * - * Copyright 2018 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.statistics; - -import com.google.protobuf.ByteString; -import org.tikv.key.Key; - -/** - * Bucket is histogram element. - * - *

Bucket bound is the smallest and greatest values stored in the bucket. The lower and upper - * bound are stored in bucket as lowerBound and upperBound. - * - *

A bucket count is the number of items stored in all previous buckets and the current bucket. - * Bucket counts are always in increasing order. - * - *

A bucket repeat is the number of repeats of the greatest bucket value, it can be used to find - * popular values. - * - *

Note that lowerBound and upperBound keys should be 'comparable objects', and these bounds are - * encoded as `binary` type in TiDB. Intuitively, you should also use Keys encoded as binary format - * to do comparison in row count estimation. - */ -public class Bucket implements Comparable { - public long count; - private long repeats; - private Key lowerBound; - private Key upperBound; - - public Bucket(long count, long repeats, ByteString lowerBound, ByteString upperBound) { - this.count = count; - this.repeats = repeats; - this.lowerBound = Key.toRawKey(lowerBound); - this.upperBound = Key.toRawKey(upperBound); - } - - public Bucket(long count, long repeats, Key lowerBound, Key upperBound) { - this.count = count; - this.repeats = repeats; - this.lowerBound = lowerBound; - this.upperBound = upperBound; - assert upperBound != null; - } - - /** used for binary search only */ - public Bucket(Key upperBound) { - this.upperBound = upperBound; - assert upperBound != null; - } - - @Override - @SuppressWarnings("unchecked") - public int compareTo(Bucket b) { - return upperBound.compareTo(b.upperBound); - } - - public long getCount() { - return count; - } - - public void setCount(long count) { - this.count = count; - } - - public long getRepeats() { - return repeats; - } - - public void setRepeats(long repeats) { - this.repeats = repeats; - } - - public Key getLowerBound() { - return lowerBound; - } - - public void setLowerBound(Key lowerBound) { - this.lowerBound = lowerBound; - } - - public Key getUpperBound() { - return upperBound; - } - - public void setUpperBound(Key upperBound) { - this.upperBound = upperBound; - } - - @Override - public String toString() { - return "{count=" - + count - + ", repeats=" - + repeats - + ", range=[" - + lowerBound - + ", " - + upperBound.toString() - + "]}"; - } -} diff --git a/src/main/java/org/tikv/statistics/CMSketch.java b/src/main/java/org/tikv/statistics/CMSketch.java deleted file mode 100644 index 4a833f16c2..0000000000 --- a/src/main/java/org/tikv/statistics/CMSketch.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2018 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.statistics; - -import com.sangupta.murmur.Murmur3; -import java.util.Arrays; - -public class CMSketch { - private int depth; - private int width; - private long count; - private long[][] table; - - public int getDepth() { - return depth; - } - - public void setDepth(int depth) { - this.depth = depth; - } - - public int getWidth() { - return width; - } - - public void setWidth(int width) { - this.width = width; - } - - public long getCount() { - return count; - } - - public void setCount(long count) { - this.count = count; - } - - public long[][] getTable() { - return table; - } - - public void setTable(long[][] table) { - this.table = table; - } - - // Hide constructor - private CMSketch() {} - - public static CMSketch newCMSketch(int d, int w) { - CMSketch sketch = new CMSketch(); - sketch.setTable(new long[d][w]); - sketch.setDepth(d); - sketch.setWidth(w); - return sketch; - } - - public long queryBytes(byte[] bytes) { - long[] randNums = Murmur3.hash_x64_128(bytes, bytes.length, 0); - long h1 = randNums[0]; - long h2 = randNums[1]; - long min = Long.MAX_VALUE; - long[] vals = new long[depth]; - for (int i = 0; i < table.length; i++) { - int j = (int) ((h1 + h2 * i) % width); - if (min > table[i][j]) { - min = table[i][j]; - } - long noise = (count - table[i][j]) / (width - 1); - if (table[i][j] < noise) { - vals[i] = 0; - } else { - vals[i] = table[i][j] - noise; - } - } - Arrays.sort(vals); - long res = vals[(depth - 1) / 2] + (vals[depth / 2] - vals[(depth - 1) / 2]) / 2; - if (res > min) { - return min; - } - return res; - } -} diff --git a/src/main/java/org/tikv/statistics/ColumnStatistics.java b/src/main/java/org/tikv/statistics/ColumnStatistics.java deleted file mode 100644 index b7ae5669ad..0000000000 --- a/src/main/java/org/tikv/statistics/ColumnStatistics.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * - * Copyright 2018 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.statistics; - -import org.tikv.meta.TiColumnInfo; - -/** - * Each Column will have a single {@link ColumnStatistics} to store {@link Histogram} info and - * {@link CMSketch} info, if any. - */ -public class ColumnStatistics { - private Histogram histogram; - private CMSketch cmSketch; - private long count; - private TiColumnInfo columnInfo; - - public ColumnStatistics( - Histogram histogram, CMSketch cmSketch, long count, TiColumnInfo columnInfo) { - this.histogram = histogram; - this.cmSketch = cmSketch; - this.count = count; - this.columnInfo = columnInfo; - } - - public Histogram getHistogram() { - return histogram; - } - - public void setHistogram(Histogram histogram) { - this.histogram = histogram; - } - - public CMSketch getCmSketch() { - return cmSketch; - } - - public void setCmSketch(CMSketch cmSketch) { - this.cmSketch = cmSketch; - } - - public long getCount() { - return count; - } - - public void setCount(long count) { - this.count = count; - } - - public TiColumnInfo getColumnInfo() { - return columnInfo; - } - - public void setColumnInfo(TiColumnInfo columnInfo) { - this.columnInfo = columnInfo; - } -} diff --git a/src/main/java/org/tikv/statistics/Histogram.java b/src/main/java/org/tikv/statistics/Histogram.java deleted file mode 100644 index bc76df9548..0000000000 --- a/src/main/java/org/tikv/statistics/Histogram.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright 2018 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.statistics; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import org.tikv.key.Key; - -/** - * Histogram represents core statistics for a column or index. - * - *

Each Histogram will have at most 256 buckets, each bucket contains a lower bound, a upper - * bound and the number of rows contain in such bound. With this information, SQL layer will be able - * to estimate how many rows will a index scan selects and determine which index to use will have - * the lowest cost. - * - * @see Bucket for moreinformation on the core data structure. - */ -public class Histogram { - - private final long id; // Column ID - private final long numberOfDistinctValue; // Number of distinct values. - private List buckets; // Histogram bucket list. - private final long nullCount; - private final long lastUpdateVersion; - - public static class Builder { - private long id; - private long NDV; - private List buckets = new ArrayList<>(); - private long nullCount; - private long lastUpdateVersion; - - public Builder setId(long id) { - this.id = id; - return this; - } - - public Builder setNDV(long NDV) { - this.NDV = NDV; - return this; - } - - public Builder setBuckets(List buckets) { - this.buckets = new ArrayList<>(buckets); - return this; - } - - public Builder setNullCount(long nullCount) { - this.nullCount = nullCount; - return this; - } - - public Builder setLastUpdateVersion(long lastUpdateVersion) { - this.lastUpdateVersion = lastUpdateVersion; - return this; - } - - public Histogram build() { - return new Histogram(id, NDV, buckets, nullCount, lastUpdateVersion); - } - } - - public static Builder newBuilder() { - return new Builder(); - } - - private Histogram( - long id, - long numberOfDistinctValue, - List buckets, - long nullCount, - long lastUpdateVersion) { - this.id = id; - this.numberOfDistinctValue = numberOfDistinctValue; - this.buckets = buckets; - this.nullCount = nullCount; - this.lastUpdateVersion = lastUpdateVersion; - } - - public long getNumberOfDistinctValue() { - return numberOfDistinctValue; - } - - public List getBuckets() { - return buckets; - } - - public long getNullCount() { - return nullCount; - } - - public long getLastUpdateVersion() { - return lastUpdateVersion; - } - - public long getId() { - return id; - } - - /** equalRowCount estimates the row count where the column equals to value. */ - double equalRowCount(Key values) { - int index = lowerBound(values); - // index not in range - if (index == -buckets.size() - 1) { - return 0; - } - // index found - if (index >= 0) { - return buckets.get(index).getRepeats(); - } - // index not found - index = -index - 1; - int cmp; - if (buckets.get(index).getLowerBound() == null) { - cmp = 1; - } else { - Objects.requireNonNull(buckets.get(index).getLowerBound()); - cmp = values.compareTo(buckets.get(index).getLowerBound()); - } - if (cmp < 0) { - return 0; - } - return totalRowCount() / numberOfDistinctValue; - } - - /** greaterRowCount estimates the row count where the column greater than value. */ - double greaterRowCount(Key values) { - double lessCount = lessRowCount(values); - double equalCount = equalRowCount(values); - double greaterCount; - greaterCount = totalRowCount() - lessCount - equalCount; - if (greaterCount < 0) { - greaterCount = 0; - } - return greaterCount; - } - - /** greaterAndEqRowCount estimates the row count where the column less than or equal to value. */ - private double greaterAndEqRowCount(Key values) { - double greaterCount = greaterRowCount(values); - double equalCount = equalRowCount(values); - return greaterCount + equalCount; - } - - /** lessRowCount estimates the row count where the column less than values. */ - double lessRowCount(Key values) { - int index = lowerBound(values); - // index not in range - if (index == -buckets.size() - 1) { - return totalRowCount(); - } - if (index < 0) { - index = -index - 1; - } else { - return buckets.get(index).count - buckets.get(index).getRepeats(); - } - double curCount = buckets.get(index).count; - double preCount = 0; - if (index > 0) { - preCount = buckets.get(index - 1).count; - } - double lessThanBucketValueCount = curCount - buckets.get(index).getRepeats(); - Key lowerBound = buckets.get(index).getLowerBound(); - int c; - if (lowerBound != null) { - c = values.compareTo(lowerBound); - } else { - c = 1; - } - if (c < 0) { - return preCount; - } - return (preCount + lessThanBucketValueCount) / 2; - } - - /** lessAndEqRowCount estimates the row count where the column less than or equal to value. */ - public double lessAndEqRowCount(Key values) { - double lessCount = lessRowCount(values); - double equalCount = equalRowCount(values); - return lessCount + equalCount; - } - - /** - * betweenRowCount estimates the row count where column greater than or equal to a and less than - * b. - */ - double betweenRowCount(Key a, Key b) { - double lessCountA = lessRowCount(a); - double lessCountB = lessRowCount(b); - // If lessCountA is not less than lessCountB, it may be that they fall to the same bucket and we - // cannot estimate - // the fraction, so we use `totalCount / NDV` to estimate the row count, but the result should - // not greater than lessCountB. - if (lessCountA >= lessCountB) { - return Math.min(lessCountB, totalRowCount() / numberOfDistinctValue); - } - return lessCountB - lessCountA; - } - - public double totalRowCount() { - if (buckets.isEmpty()) { - return 0; - } - return buckets.get(buckets.size() - 1).count + nullCount; - } - - /** - * lowerBound returns the smallest index of the searched key and returns (-[insertion point] - 1) - * if the key is not found in buckets where [insertion point] denotes the index of the first - * element greater than the key - */ - public int lowerBound(Key key) { - assert key.getClass() == buckets.get(0).getUpperBound().getClass(); - return Arrays.binarySearch(buckets.toArray(), new Bucket(key)); - } - - /** - * mergeBuckets is used to merge every two neighbor buckets. - * - * @param bucketIdx: index of the last bucket. - */ - public void mergeBlock(int bucketIdx) { - int curBuck = 0; - for (int i = 0; i + 1 <= bucketIdx; i += 2) { - buckets.set( - curBuck++, - new Bucket( - buckets.get(i + 1).count, - buckets.get(i + 1).getRepeats(), - buckets.get(i + 1).getLowerBound(), - buckets.get(i).getUpperBound())); - } - if (bucketIdx % 2 == 0) { - buckets.set(curBuck++, buckets.get(bucketIdx)); - } - buckets = buckets.subList(0, curBuck); - } - - /** getIncreaseFactor will return a factor of data increasing after the last analysis. */ - double getIncreaseFactor(long totalCount) { - long columnCount = buckets.get(buckets.size() - 1).count + nullCount; - if (columnCount == 0) { - return 1.0; - } - return (double) totalCount / (double) columnCount; - } -} diff --git a/src/main/java/org/tikv/statistics/IndexStatistics.java b/src/main/java/org/tikv/statistics/IndexStatistics.java deleted file mode 100644 index f7a95d8244..0000000000 --- a/src/main/java/org/tikv/statistics/IndexStatistics.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2018 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.statistics; - -import com.google.common.collect.BoundType; -import com.google.common.collect.Range; -import java.util.List; -import org.tikv.codec.CodecDataOutput; -import org.tikv.key.Key; -import org.tikv.key.TypedKey; -import org.tikv.meta.TiIndexInfo; -import org.tikv.predicates.IndexRange; -import org.tikv.types.DataTypeFactory; -import org.tikv.types.MySQLType; - -/** - * Each Index will have a single {@link IndexStatistics} to store {@link Histogram} info and {@link - * CMSketch} info, if any. - */ -public class IndexStatistics { - private Histogram histogram; - private CMSketch cmSketch; - private TiIndexInfo indexInfo; - - public IndexStatistics(Histogram histogram, CMSketch cmSketch, TiIndexInfo indexInfo) { - this.histogram = histogram; - this.cmSketch = cmSketch; - this.indexInfo = indexInfo; - } - - public Histogram getHistogram() { - return histogram; - } - - public void setHistogram(Histogram histogram) { - this.histogram = histogram; - } - - public CMSketch getCmSketch() { - return cmSketch; - } - - public void setCmSketch(CMSketch cmSketch) { - this.cmSketch = cmSketch; - } - - public TiIndexInfo getIndexInfo() { - return indexInfo; - } - - public void setIndexInfo(TiIndexInfo indexInfo) { - this.indexInfo = indexInfo; - } - - public double getRowCount(List indexRanges) { - double rowCount = 0.0; - for (IndexRange ir : indexRanges) { - double cnt = 0.0; - Key pointKey = ir.hasAccessKey() ? ir.getAccessKey() : Key.EMPTY; - Range range = ir.getRange(); - Key lPointKey; - Key uPointKey; - - Key lKey; - Key uKey; - if (pointKey != Key.EMPTY) { - Key convertedKey = - TypedKey.toTypedKey(pointKey.getBytes(), DataTypeFactory.of(MySQLType.TypeBlob)); - Key convertedNext = - TypedKey.toTypedKey(pointKey.next().getBytes(), DataTypeFactory.of(MySQLType.TypeBlob)); - // TODO: Implement CMSketch point query - // if (cmSketch != null) { - // rowCount += cmSketch.queryBytes(convertedKey.getBytes()); - // } else { - rowCount += histogram.betweenRowCount(convertedKey, convertedNext); - // } - } - if (range != null) { - lPointKey = pointKey; - uPointKey = pointKey; - - if (!range.hasLowerBound()) { - // -INF - lKey = Key.MIN; - } else { - lKey = range.lowerEndpoint(); - if (range.lowerBoundType().equals(BoundType.OPEN)) { - lKey = lKey.next(); - } - } - if (!range.hasUpperBound()) { - // INF - uKey = Key.MAX; - } else { - uKey = range.upperEndpoint(); - if (range.upperBoundType().equals(BoundType.CLOSED)) { - uKey = uKey.next(); - } - } - CodecDataOutput cdo = new CodecDataOutput(); - cdo.write(lPointKey.getBytes()); - cdo.write(lKey.getBytes()); - Key lowerBound = TypedKey.toTypedKey(cdo.toBytes(), DataTypeFactory.of(MySQLType.TypeBlob)); - cdo.reset(); - cdo.write(uPointKey.getBytes()); - cdo.write(uKey.getBytes()); - Key upperBound = TypedKey.toTypedKey(cdo.toBytes(), DataTypeFactory.of(MySQLType.TypeBlob)); - cnt += histogram.betweenRowCount(lowerBound, upperBound); - } - rowCount += cnt; - } - if (rowCount > histogram.totalRowCount()) { - rowCount = histogram.totalRowCount(); - } else if (rowCount < 0) { - rowCount = 0; - } - return rowCount; - } -} diff --git a/src/main/java/org/tikv/statistics/TableStatistics.java b/src/main/java/org/tikv/statistics/TableStatistics.java deleted file mode 100644 index 270813bb92..0000000000 --- a/src/main/java/org/tikv/statistics/TableStatistics.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * - * Copyright 2018 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.statistics; - -import java.util.HashMap; -import java.util.Map; - -/** - * A TableStatistics Java plain object. - * - *

Usually each table will have two types of statistics information: 1. Meta info (tableId, - * count, modifyCount, version) 2. Column/Index histogram info (columnsHistMap, indexHistMap) - */ -public class TableStatistics { - private final long tableId; // Which table it belongs to - private final Map columnsHistMap = - new HashMap<>(); // ColumnId -> ColumnStatistics map - private final Map indexHistMap = - new HashMap<>(); // IndexId -> IndexStatistics map - private long count; // Total row count in a table. - private long modifyCount; // Total modify count in a table. - private long version; // Version of this statistics info - - public TableStatistics(long tableId) { - this.tableId = tableId; - } - - public long getTableId() { - return tableId; - } - - public Map getColumnsHistMap() { - return columnsHistMap; - } - - public Map getIndexHistMap() { - return indexHistMap; - } - - public long getCount() { - return count; - } - - public void setCount(long count) { - this.count = count; - } - - public long getModifyCount() { - return modifyCount; - } - - public void setModifyCount(long modifyCount) { - this.modifyCount = modifyCount; - } - - public long getVersion() { - return version; - } - - public void setVersion(long version) { - this.version = version; - } -} diff --git a/src/main/java/org/tikv/tools/RegionUtils.java b/src/main/java/org/tikv/tools/RegionUtils.java deleted file mode 100644 index 79a614ace6..0000000000 --- a/src/main/java/org/tikv/tools/RegionUtils.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.tools; - -import static java.util.Objects.requireNonNull; - -import com.google.common.collect.ImmutableList; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.tikv.TiSession; -import org.tikv.meta.TiTableInfo; -import org.tikv.predicates.ScanAnalyzer; -import org.tikv.util.RangeSplitter; -import org.tikv.util.RangeSplitter.RegionTask; - -public class RegionUtils { - public static Map getRegionDistribution( - TiSession session, String databaseName, String tableName) { - List tasks = getRegionTasks(session, databaseName, tableName); - Map regionMap = new HashMap<>(); - for (RegionTask task : tasks) { - regionMap.merge(task.getHost() + "_" + task.getStore().getId(), 1, Integer::sum); - } - return regionMap; - } - - public static Map> getStoreRegionIdDistribution( - TiSession session, String databaseName, String tableName) { - List tasks = getRegionTasks(session, databaseName, tableName); - Map> storeMap = new HashMap<>(); - for (RegionTask task : tasks) { - long regionId = task.getRegion().getId(); - long storeId = task.getStore().getId(); - storeMap.putIfAbsent(storeId, new ArrayList<>()); - storeMap.get(storeId).add(regionId); - } - return storeMap; - } - - private static List getRegionTasks( - TiSession session, String databaseName, String tableName) { - requireNonNull(session, "session is null"); - requireNonNull(databaseName, "databaseName is null"); - requireNonNull(tableName, "tableName is null"); - TiTableInfo table = session.getCatalog().getTable(databaseName, tableName); - requireNonNull(table, String.format("Table not found %s.%s", databaseName, tableName)); - ScanAnalyzer builder = new ScanAnalyzer(); - ScanAnalyzer.ScanPlan scanPlan = - builder.buildScan(ImmutableList.of(), ImmutableList.of(), table); - return RangeSplitter.newSplitter(session.getRegionManager()) - .splitRangeByRegion(scanPlan.getKeyRanges()); - } -} diff --git a/src/main/java/org/tikv/types/AbstractDateTimeType.java b/src/main/java/org/tikv/types/AbstractDateTimeType.java deleted file mode 100644 index 1e86fada0d..0000000000 --- a/src/main/java/org/tikv/types/AbstractDateTimeType.java +++ /dev/null @@ -1,82 +0,0 @@ -package org.tikv.types; - -import com.pingcap.tidb.tipb.ExprType; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.LocalDate; -import org.tikv.codec.Codec; -import org.tikv.codec.Codec.DateCodec; -import org.tikv.codec.Codec.DateTimeCodec; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.InvalidCodecFormatException; -import org.tikv.meta.TiColumnInfo.InternalTypeHolder; - -public abstract class AbstractDateTimeType extends DataType { - AbstractDateTimeType(InternalTypeHolder holder) { - super(holder); - } - - AbstractDateTimeType(MySQLType tp) { - super(tp); - } - - /** Return timezone used for encoding and decoding */ - protected abstract DateTimeZone getTimezone(); - - /** - * Decode DateTime from packed long value In TiDB / MySQL, timestamp type is converted to UTC and - * stored - */ - DateTime decodeDateTime(int flag, CodecDataInput cdi) { - DateTime dateTime; - if (flag == Codec.UVARINT_FLAG) { - dateTime = DateTimeCodec.readFromUVarInt(cdi, getTimezone()); - } else if (flag == Codec.UINT_FLAG) { - dateTime = DateTimeCodec.readFromUInt(cdi, getTimezone()); - } else { - throw new InvalidCodecFormatException( - "Invalid Flag type for " + getClass().getSimpleName() + ": " + flag); - } - return dateTime; - } - - /** Decode Date from packed long value */ - LocalDate decodeDate(int flag, CodecDataInput cdi) { - LocalDate date; - if (flag == Codec.UVARINT_FLAG) { - date = DateCodec.readFromUVarInt(cdi); - } else if (flag == Codec.UINT_FLAG) { - date = DateCodec.readFromUInt(cdi); - } else { - throw new InvalidCodecFormatException( - "Invalid Flag type for " + getClass().getSimpleName() + ": " + flag); - } - return date; - } - - /** {@inheritDoc} */ - @Override - protected void encodeKey(CodecDataOutput cdo, Object value) { - DateTime dt = Converter.convertToDateTime(value); - DateTimeCodec.writeDateTimeFully(cdo, dt, getTimezone()); - } - - /** {@inheritDoc} */ - @Override - protected void encodeValue(CodecDataOutput cdo, Object value) { - encodeKey(cdo, value); - } - - /** {@inheritDoc} */ - @Override - protected void encodeProto(CodecDataOutput cdo, Object value) { - DateTime dt = Converter.convertToDateTime(value); - DateTimeCodec.writeDateTimeProto(cdo, dt, getTimezone()); - } - - @Override - public ExprType getProtoExprType() { - return ExprType.MysqlTime; - } -} diff --git a/src/main/java/org/tikv/types/BitType.java b/src/main/java/org/tikv/types/BitType.java deleted file mode 100644 index bdbf6c8624..0000000000 --- a/src/main/java/org/tikv/types/BitType.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import org.tikv.codec.Codec; -import org.tikv.codec.Codec.IntegerCodec; -import org.tikv.codec.CodecDataInput; -import org.tikv.exception.TypeException; -import org.tikv.meta.TiColumnInfo; - -public class BitType extends IntegerType { - public static final BitType BIT = new BitType(MySQLType.TypeBit); - - public static final MySQLType[] subTypes = new MySQLType[] {MySQLType.TypeBit}; - - private BitType(MySQLType tp) { - super(tp); - } - - protected BitType(TiColumnInfo.InternalTypeHolder holder) { - super(holder); - } - - /** {@inheritDoc} */ - @Override - protected Object decodeNotNull(int flag, CodecDataInput cdi) { - switch (flag) { - case Codec.UVARINT_FLAG: - return IntegerCodec.readUVarLong(cdi); - case Codec.UINT_FLAG: - return IntegerCodec.readULong(cdi); - default: - throw new TypeException("Invalid IntegerType flag: " + flag); - } - } - - @Override - public boolean isUnsigned() { - return true; - } -} diff --git a/src/main/java/org/tikv/types/BytesType.java b/src/main/java/org/tikv/types/BytesType.java deleted file mode 100644 index 3315ea78f6..0000000000 --- a/src/main/java/org/tikv/types/BytesType.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import com.pingcap.tidb.tipb.ExprType; -import org.tikv.codec.Codec; -import org.tikv.codec.Codec.BytesCodec; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.InvalidCodecFormatException; -import org.tikv.meta.TiColumnInfo; - -/** - * TODO: if we need to unify string type and binary types? Indeed they are encoded as the same - * However, decode to string actually going through encoding/decoding by whatever charset.encoding - * format we set, and essentially changed underlying data - */ -public class BytesType extends DataType { - public static final BytesType BLOB = new BytesType(MySQLType.TypeBlob); - public static final BytesType LONG_TEXT = new BytesType(MySQLType.TypeLongBlob); - public static final BytesType MEDIUM_TEXT = new BytesType(MySQLType.TypeMediumBlob); - public static final BytesType TEXT = new BytesType(MySQLType.TypeBlob); - public static final BytesType TINY_BLOB = new BytesType(MySQLType.TypeTinyBlob); - - public static final MySQLType[] subTypes = - new MySQLType[] { - MySQLType.TypeBlob, MySQLType.TypeLongBlob, - MySQLType.TypeMediumBlob, MySQLType.TypeTinyBlob - }; - - protected BytesType(MySQLType tp) { - super(tp); - } - - protected BytesType(TiColumnInfo.InternalTypeHolder holder) { - super(holder); - } - - /** {@inheritDoc} */ - @Override - protected Object decodeNotNull(int flag, CodecDataInput cdi) { - if (flag == Codec.COMPACT_BYTES_FLAG) { - return BytesCodec.readCompactBytes(cdi); - } else if (flag == Codec.BYTES_FLAG) { - return BytesCodec.readBytes(cdi); - } else { - throw new InvalidCodecFormatException("Invalid Flag type for : " + flag); - } - } - - @Override - protected boolean isPrefixIndexSupported() { - return true; - } - - /** {@inheritDoc} */ - @Override - protected void encodeKey(CodecDataOutput cdo, Object value) { - byte[] bytes = Converter.convertToBytes(value); - BytesCodec.writeBytesFully(cdo, bytes); - } - - /** {@inheritDoc} */ - @Override - protected void encodeValue(CodecDataOutput cdo, Object value) { - byte[] bytes = Converter.convertToBytes(value); - BytesCodec.writeCompactBytesFully(cdo, bytes); - } - - /** {@inheritDoc} */ - @Override - protected void encodeProto(CodecDataOutput cdo, Object value) { - byte[] bytes = Converter.convertToBytes(value); - BytesCodec.writeBytesRaw(cdo, bytes); - } - - @Override - public ExprType getProtoExprType() { - return getCharset().equals(Charset.CharsetBin) ? ExprType.Bytes : ExprType.String; - } - - /** {@inheritDoc} */ - @Override - public Object getOriginDefaultValueNonNull(String value) { - return value; - } -} diff --git a/src/main/java/org/tikv/types/Charset.java b/src/main/java/org/tikv/types/Charset.java deleted file mode 100644 index bb770401b1..0000000000 --- a/src/main/java/org/tikv/types/Charset.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.types; - -public class Charset { - // CharsetBin is used for marking binary charset. - public static final String CharsetBin = "binary"; - // CollationBin is the default collation for CharsetBin. - public static final String CollationBin = "binary"; - // CharsetUTF8 is the default charset for string types. - public static final String CharsetUTF8 = "utf8"; - // CollationUTF8 is the default collation for CharsetUTF8. - public static final String CollationUTF8 = "utf8_bin"; - // CharsetUTF8MB4 represents 4 bytes utf8, which works the same way as utf8 in Go. - public static final String CharsetUTF8MB4 = "utf8mb4"; - // CollationUTF8MB4 is the default collation for CharsetUTF8MB4. - public static final String CollationUTF8MB4 = "utf8mb4_bin"; - // CharsetASCII is a subset of UTF8. - public static final String CharsetASCII = "ascii"; - // CollationASCII is the default collation for CharsetACSII. - public static final String CollationASCII = "ascii_bin"; - // CharsetLatin1 is a single byte charset. - public static final String CharsetLatin1 = "latin1"; - // CollationLatin1 is the default collation for CharsetLatin1. - public static final String CollationLatin1 = "latin1_bin"; -} diff --git a/src/main/java/org/tikv/types/Converter.java b/src/main/java/org/tikv/types/Converter.java deleted file mode 100644 index d40e48b086..0000000000 --- a/src/main/java/org/tikv/types/Converter.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import static java.util.Objects.requireNonNull; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.sql.Date; -import java.sql.Timestamp; -import java.util.Arrays; -import org.apache.spark.unsafe.types.UTF8String; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; -import org.tikv.exception.TypeException; - -public class Converter { - public static long convertToLong(Object val) { - requireNonNull(val, "val is null"); - if (val instanceof Number) { - return ((Number) val).longValue(); - } else if (val instanceof String) { - return Long.parseLong(val.toString()); - } - throw new TypeException( - String.format("Cannot cast %s to long", val.getClass().getSimpleName())); - } - - public static double convertToDouble(Object val) { - requireNonNull(val, "val is null"); - if (val instanceof Number) { - return ((Number) val).doubleValue(); - } else if (val instanceof String) { - return Double.parseDouble(val.toString()); - } - throw new TypeException( - String.format("Cannot cast %s to double", val.getClass().getSimpleName())); - } - - public static String convertToString(Object val) { - requireNonNull(val, "val is null"); - return val.toString(); - } - - public static byte[] convertToBytes(Object val) { - requireNonNull(val, "val is null"); - if (val instanceof byte[]) { - return (byte[]) val; - } else if (val instanceof String) { - return ((String) val).getBytes(); - } - throw new TypeException( - String.format("Cannot cast %s to bytes", val.getClass().getSimpleName())); - } - - static byte[] convertToBytes(Object val, int prefixLength) { - requireNonNull(val, "val is null"); - if (val instanceof byte[]) { - return Arrays.copyOf((byte[]) val, prefixLength); - } else if (val instanceof String) { - return Arrays.copyOf(((String) val).getBytes(), prefixLength); - } - throw new TypeException( - String.format("Cannot cast %s to bytes", val.getClass().getSimpleName())); - } - - static byte[] convertUtf8ToBytes(Object val, int prefixLength) { - requireNonNull(val, "val is null"); - if (val instanceof byte[]) { - return UTF8String.fromBytes(((byte[]) val)).substring(0, prefixLength).getBytes(); - } else if (val instanceof String) { - return UTF8String.fromString(((String) val)).substring(0, prefixLength).getBytes(); - } - throw new TypeException( - String.format("Cannot cast %s to bytes", val.getClass().getSimpleName())); - } - - private static final DateTimeZone localTimeZone = DateTimeZone.getDefault(); - private static final DateTimeFormatter localDateTimeFormatter = - DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZone(localTimeZone); - private static final DateTimeFormatter localDateFormatter = - DateTimeFormat.forPattern("yyyy-MM-dd").withZone(localTimeZone); - - public static DateTimeZone getLocalTimezone() { - return localTimeZone; - } - - /** - * Convert an object to Datetime If constant is a string, it parses as local timezone If it is an - * long, it parsed as UTC epoch - * - * @param val value to be converted to DateTime - * @return joda.time.DateTime indicating local Datetime - */ - public static DateTime convertToDateTime(Object val) { - requireNonNull(val, "val is null"); - if (val instanceof DateTime) { - return (DateTime) val; - } else if (val instanceof String) { - // interpret string as in local timezone - try { - return DateTime.parse((String) val, localDateTimeFormatter); - } catch (Exception e) { - throw new TypeException( - String.format("Error parsing string %s to datetime", (String) val), e); - } - } else if (val instanceof Long) { - return new DateTime((long) val); - } else if (val instanceof Timestamp) { - return new DateTime(((Timestamp) val).getTime()); - } else if (val instanceof Date) { - return new DateTime(((Date) val).getTime()); - } else { - throw new TypeException("Can not cast Object to LocalDateTime "); - } - } - - /** - * Convert an object to Date If constant is a string, it parses as local timezone If it is an - * long, it parsed as UTC epoch - * - * @param val value to be converted to DateTime - * @return java.sql.Date indicating Date - */ - public static Date convertToDate(Object val) { - requireNonNull(val, "val is null"); - if (val instanceof Date) { - return (Date) val; - } else if (val instanceof String) { - try { - return new Date(DateTime.parse((String) val, localDateFormatter).toDate().getTime()); - } catch (Exception e) { - throw new TypeException(String.format("Error parsing string %s to date", (String) val), e); - } - } else if (val instanceof Long) { - return new Date((long) val); - } else if (val instanceof Timestamp) { - return new Date(((Timestamp) val).getTime()); - } else if (val instanceof DateTime) { - return new Date(((DateTime) val).getMillis()); - } else { - throw new TypeException("Can not cast Object to LocalDate"); - } - } - - public static BigDecimal convertToBigDecimal(Object val) { - requireNonNull(val, "val is null"); - if (val instanceof BigDecimal) { - return (BigDecimal) val; - } else if (val instanceof Double || val instanceof Float) { - return new BigDecimal((Double) val); - } else if (val instanceof BigInteger) { - return new BigDecimal((BigInteger) val); - } else if (val instanceof Number) { - return new BigDecimal(((Number) val).longValue()); - } else if (val instanceof String) { - return new BigDecimal((String) val); - } else { - throw new TypeException("can not cast non Number type to Double"); - } - } -} diff --git a/src/main/java/org/tikv/types/DataType.java b/src/main/java/org/tikv/types/DataType.java deleted file mode 100644 index 1feddb2007..0000000000 --- a/src/main/java/org/tikv/types/DataType.java +++ /dev/null @@ -1,361 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.types; - -import static java.util.Objects.requireNonNull; -import static org.tikv.codec.Codec.isNullFlag; - -import com.google.common.collect.ImmutableList; -import com.pingcap.tidb.tipb.ExprType; -import java.io.Serializable; -import java.util.List; -import org.tikv.codec.Codec; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.TypeException; -import org.tikv.meta.Collation; -import org.tikv.meta.TiColumnInfo; -import org.tikv.meta.TiColumnInfo.InternalTypeHolder; - -/** Base Type for encoding and decoding TiDB row information. */ -public abstract class DataType implements Serializable { - - // Flag Information for strict mysql type - public static final int NotNullFlag = 1; /* Field can't be NULL */ - public static final int PriKeyFlag = 2; /* Field is part of a primary key */ - public static final int UniqueKeyFlag = 4; /* Field is part of a unique key */ - public static final int MultipleKeyFlag = 8; /* Field is part of a key */ - public static final int BlobFlag = 16; /* Field is a blob */ - public static final int UnsignedFlag = 32; /* Field is unsigned */ - public static final int ZerofillFlag = 64; /* Field is zerofill */ - public static final int BinaryFlag = 128; /* Field is binary */ - public static final int EnumFlag = 256; /* Field is an enum */ - public static final int AutoIncrementFlag = 512; /* Field is an auto increment field */ - public static final int TimestampFlag = 1024; /* Field is a timestamp */ - public static final int SetFlag = 2048; /* Field is a set */ - public static final int NoDefaultValueFlag = 4096; /* Field doesn't have a default value */ - public static final int OnUpdateNowFlag = 8192; /* Field is set to NOW on UPDATE */ - public static final int NumFlag = 32768; /* Field is a num (for clients) */ - - public enum EncodeType { - KEY, - VALUE, - PROTO - } - - public static final int UNSPECIFIED_LEN = -1; - - // MySQL type - protected final MySQLType tp; - // Not Encode/Decode flag, this is used to strict mysql type - // such as not null, timestamp - protected final int flag; - protected final int decimal; - private final String charset; - protected final int collation; - protected final long length; - private final List elems; - - protected DataType(TiColumnInfo.InternalTypeHolder holder) { - this.tp = MySQLType.fromTypeCode(holder.getTp()); - this.flag = holder.getFlag(); - this.length = holder.getFlen(); - this.decimal = holder.getDecimal(); - this.charset = holder.getCharset(); - this.collation = Collation.translate(holder.getCollate()); - this.elems = holder.getElems() == null ? ImmutableList.of() : holder.getElems(); - } - - protected DataType(MySQLType type) { - this.tp = type; - this.flag = 0; - this.elems = ImmutableList.of(); - this.length = UNSPECIFIED_LEN; - this.decimal = UNSPECIFIED_LEN; - this.charset = ""; - this.collation = Collation.DEF_COLLATION_CODE; - } - - protected DataType( - MySQLType type, int flag, int len, int decimal, String charset, int collation) { - this.tp = type; - this.flag = flag; - this.elems = ImmutableList.of(); - this.length = len; - this.decimal = decimal; - this.charset = charset; - this.collation = collation; - } - - protected abstract Object decodeNotNull(int flag, CodecDataInput cdi); - - /** - * decode value from row which is nothing. - * - * @param cdi source of data. - */ - public Object decode(CodecDataInput cdi) { - int flag = cdi.readUnsignedByte(); - if (isNullFlag(flag)) { - return null; - } - return decodeNotNull(flag, cdi); - } - - public boolean isNextNull(CodecDataInput cdi) { - return isNullFlag(cdi.peekByte()); - } - - public static void encodeMaxValue(CodecDataOutput cdo) { - cdo.writeByte(Codec.MAX_FLAG); - } - - public static void encodeNull(CodecDataOutput cdo) { - cdo.writeByte(Codec.NULL_FLAG); - } - - public static void encodeIndex(CodecDataOutput cdo) { - cdo.writeByte(Codec.BYTES_FLAG); - } - - /** - * encode a Row to CodecDataOutput - * - * @param cdo destination of data. - * @param encodeType Key or Value. - * @param value value to be encoded. - */ - public void encode(CodecDataOutput cdo, EncodeType encodeType, Object value) { - requireNonNull(cdo, "cdo is null"); - if (value == null) { - if (encodeType != EncodeType.PROTO) { - encodeNull(cdo); - } - } else { - switch (encodeType) { - case KEY: - encodeKey(cdo, value); - return; - case VALUE: - encodeValue(cdo, value); - return; - case PROTO: - encodeProto(cdo, value); - return; - default: - throw new TypeException("Unknown encoding type " + encodeType); - } - } - } - - protected abstract void encodeKey(CodecDataOutput cdo, Object value); - - protected abstract void encodeValue(CodecDataOutput cdo, Object value); - - protected abstract void encodeProto(CodecDataOutput cdo, Object value); - - /** - * encode a Key's prefix to CodecDataOutput - * - * @param cdo destination of data. - * @param value value to be encoded. - * @param type data value type. - * @param prefixLength specifies prefix length of value to be encoded. When prefixLength is - * DataType.UNSPECIFIED_LEN, encode full length of value. - */ - public void encodeKey(CodecDataOutput cdo, Object value, DataType type, int prefixLength) { - requireNonNull(cdo, "cdo is null"); - if (value == null) { - encodeNull(cdo); - } else if (prefixLength == DataType.UNSPECIFIED_LEN) { - encodeKey(cdo, value); - } else if (isPrefixIndexSupported()) { - byte[] bytes; - // When charset is utf8/utf8mb4, prefix length should be the number of utf8 characters - // rather than length of its encoded byte value. - if (type.getCharset().equalsIgnoreCase("utf8") - || type.getCharset().equalsIgnoreCase("utf8mb4")) { - bytes = Converter.convertUtf8ToBytes(value, prefixLength); - } else { - bytes = Converter.convertToBytes(value, prefixLength); - } - Codec.BytesCodec.writeBytesFully(cdo, bytes); - } else { - throw new TypeException("Data type can not encode with prefix"); - } - } - - /** - * Indicates whether a data type supports prefix index - * - * @return returns true iff the type is BytesType - */ - protected boolean isPrefixIndexSupported() { - return false; - } - - public abstract ExprType getProtoExprType(); - - /** - * get origin default value - * - * @param value a int value represents in string - * @return a int object - */ - public abstract Object getOriginDefaultValueNonNull(String value); - - public Object getOriginDefaultValue(String value) { - if (value == null) return null; - return getOriginDefaultValueNonNull(value); - } - - public int getCollationCode() { - return collation; - } - - public long getLength() { - return length; - } - - public int getDecimal() { - return decimal; - } - - public int getFlag() { - return flag; - } - - public List getElems() { - return this.elems; - } - - public int getTypeCode() { - return tp.getTypeCode(); - } - - public MySQLType getType() { - return tp; - } - - public String getCharset() { - return charset; - } - - public boolean isPrimaryKey() { - return (flag & PriKeyFlag) > 0; - } - - public boolean isNotNull() { - return (flag & NotNullFlag) > 0; - } - - public boolean isNoDefault() { - return (flag & NoDefaultValueFlag) > 0; - } - - public boolean isAutoIncrement() { - return (flag & AutoIncrementFlag) > 0; - } - - public boolean isZeroFill() { - return (flag & ZerofillFlag) > 0; - } - - public boolean isBinary() { - return (flag & BinaryFlag) > 0; - } - - public boolean isUniqueKey() { - return (flag & UniqueKeyFlag) > 0; - } - - public boolean isMultiKey() { - return (flag & MultipleKeyFlag) > 0; - } - - public boolean isTimestamp() { - return (flag & TimestampFlag) > 0; - } - - public boolean isOnUpdateNow() { - return (flag & OnUpdateNowFlag) > 0; - } - - public boolean isBlob() { - return (flag & BlobFlag) > 0; - } - - public boolean isEnum() { - return (flag & EnumFlag) > 0; - } - - public boolean isSet() { - return (flag & SetFlag) > 0; - } - - public boolean isNum() { - return (flag & NumFlag) > 0; - } - - @Override - public String toString() { - return String.format("%s:%s", this.getClass().getSimpleName(), getType()); - } - - @Override - public boolean equals(Object other) { - if (other instanceof DataType) { - DataType otherType = (DataType) other; - // tp implies Class is the same - // and that might not always hold - // TODO: reconsider design here - return tp == otherType.tp - && flag == otherType.flag - && decimal == otherType.decimal - && (charset != null && charset.equals(otherType.charset)) - && collation == otherType.collation - && length == otherType.length - && elems.equals(otherType.elems); - } - return false; - } - - @Override - public int hashCode() { - return (int) - (31 - * (tp.getTypeCode() == 0 ? 1 : tp.getTypeCode()) - * (flag == 0 ? 1 : flag) - * (decimal == 0 ? 1 : decimal) - * (charset == null ? 1 : charset.hashCode()) - * (collation == 0 ? 1 : collation) - * (length == 0 ? 1 : length) - * (elems.hashCode())); - } - - public InternalTypeHolder toTypeHolder() { - return new InternalTypeHolder( - getTypeCode(), - flag, - length, - decimal, - charset, - "", - "", - Collation.translate(collation), - elems); - } -} diff --git a/src/main/java/org/tikv/types/DataTypeFactory.java b/src/main/java/org/tikv/types/DataTypeFactory.java deleted file mode 100644 index 332772e5a6..0000000000 --- a/src/main/java/org/tikv/types/DataTypeFactory.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import com.google.common.collect.ImmutableMap; -import java.lang.reflect.Constructor; -import java.util.Arrays; -import java.util.Map; -import org.tikv.exception.TypeException; -import org.tikv.meta.TiColumnInfo.InternalTypeHolder; - -public class DataTypeFactory { - private static final Map> dataTypeCreatorMap; - private static final Map dataTypeInstanceMap; - - static { - ImmutableMap.Builder> builder = - ImmutableMap.builder(); - ImmutableMap.Builder instBuilder = ImmutableMap.builder(); - extractTypeMap(BitType.subTypes, BitType.class, builder, instBuilder); - extractTypeMap(StringType.subTypes, StringType.class, builder, instBuilder); - extractTypeMap(DateTimeType.subTypes, DateTimeType.class, builder, instBuilder); - extractTypeMap(DateType.subTypes, DateType.class, builder, instBuilder); - extractTypeMap(DecimalType.subTypes, DecimalType.class, builder, instBuilder); - extractTypeMap(IntegerType.subTypes, IntegerType.class, builder, instBuilder); - extractTypeMap(BytesType.subTypes, BytesType.class, builder, instBuilder); - extractTypeMap(RealType.subTypes, RealType.class, builder, instBuilder); - extractTypeMap(TimestampType.subTypes, TimestampType.class, builder, instBuilder); - extractTypeMap(EnumType.subTypes, EnumType.class, builder, instBuilder); - extractTypeMap(SetType.subTypes, SetType.class, builder, instBuilder); - extractTypeMap(YearType.subTypes, YearType.class, builder, instBuilder); - extractTypeMap(JsonType.subTypes, JsonType.class, builder, instBuilder); - dataTypeCreatorMap = builder.build(); - dataTypeInstanceMap = instBuilder.build(); - } - - private static void extractTypeMap( - MySQLType[] types, - Class cls, - ImmutableMap.Builder> holderBuilder, - ImmutableMap.Builder instBuilder) { - for (MySQLType type : types) { - try { - Constructor ctorByHolder = cls.getDeclaredConstructor(InternalTypeHolder.class); - Constructor ctorByType = cls.getDeclaredConstructor(MySQLType.class); - ctorByHolder.setAccessible(true); - ctorByType.setAccessible(true); - holderBuilder.put(type, ctorByHolder); - instBuilder.put(type, (DataType) ctorByType.newInstance(type)); - } catch (Exception e) { - throw new TypeException( - String.format("Type %s does not have a proper constructor", cls.getName()), e); - } - } - } - - public static DataType of(MySQLType type) { - DataType dataType = dataTypeInstanceMap.get(type); - if (dataType == null) { - throw new TypeException("Type not found for " + type); - } - return dataType; - } - - // Convert non-binary to string type - private static MySQLType convertType(MySQLType type, InternalTypeHolder holder) { - if (Arrays.asList(BytesType.subTypes).contains(type) - && !Charset.CharsetBin.equals(holder.getCharset())) { - return MySQLType.TypeVarchar; - } - if (Arrays.asList(StringType.subTypes).contains(type) - && Charset.CharsetBin.equals(holder.getCharset())) { - return MySQLType.TypeBlob; - } - return type; - } - - public static DataType of(InternalTypeHolder holder) { - MySQLType type = MySQLType.fromTypeCode(holder.getTp()); - type = convertType(type, holder); - Constructor ctor = dataTypeCreatorMap.get(type); - if (ctor == null) { - throw new NullPointerException( - "tp " + holder.getTp() + " passed in can not retrieved DataType info."); - } - try { - return ctor.newInstance(holder); - } catch (Exception e) { - throw new TypeException("Cannot create type from " + holder.getTp(), e); - } - } -} diff --git a/src/main/java/org/tikv/types/DateTimeType.java b/src/main/java/org/tikv/types/DateTimeType.java deleted file mode 100644 index b56bc6b02f..0000000000 --- a/src/main/java/org/tikv/types/DateTimeType.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import java.sql.Timestamp; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.tikv.codec.CodecDataInput; -import org.tikv.meta.TiColumnInfo; - -/** - * Datetime is a timezone neutral version of timestamp While most of decoding logic is the same it - * interpret as local timezone to be able to compute with date/time data - */ -public class DateTimeType extends AbstractDateTimeType { - public static final DateTimeType DATETIME = new DateTimeType(MySQLType.TypeDatetime); - public static final MySQLType[] subTypes = new MySQLType[] {MySQLType.TypeDatetime}; - - private DateTimeType(MySQLType tp) { - super(tp); - } - - DateTimeType(TiColumnInfo.InternalTypeHolder holder) { - super(holder); - } - - @Override - protected DateTimeZone getTimezone() { - return Converter.getLocalTimezone(); - } - - /** - * Decode timestamp from packed long value In TiDB / MySQL, timestamp type is converted to UTC and - * stored - */ - @Override - protected Timestamp decodeNotNull(int flag, CodecDataInput cdi) { - DateTime dateTime = decodeDateTime(flag, cdi); - // Even though null is filtered out but data like 0000-00-00 exists - // according to MySQL JDBC behavior, it's converted to null - if (dateTime == null) { - return null; - } - return new Timestamp(dateTime.getMillis()); - } - - @Override - public DateTime getOriginDefaultValueNonNull(String value) { - return Converter.convertToDateTime(value); - } -} diff --git a/src/main/java/org/tikv/types/DateType.java b/src/main/java/org/tikv/types/DateType.java deleted file mode 100644 index f9a0c058e0..0000000000 --- a/src/main/java/org/tikv/types/DateType.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import java.sql.Date; -import org.joda.time.DateTimeZone; -import org.joda.time.LocalDate; -import org.tikv.codec.Codec; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.meta.TiColumnInfo; - -public class DateType extends AbstractDateTimeType { - public static final DateType DATE = new DateType(MySQLType.TypeDate); - public static final MySQLType[] subTypes = new MySQLType[] {MySQLType.TypeDate}; - - private DateType(MySQLType tp) { - super(tp); - } - - DateType(TiColumnInfo.InternalTypeHolder holder) { - super(holder); - } - - @Override - protected DateTimeZone getTimezone() { - return Converter.getLocalTimezone(); - } - - @Override - public Date getOriginDefaultValueNonNull(String value) { - return Converter.convertToDate(value); - } - - /** {@inheritDoc} */ - @Override - protected void encodeKey(CodecDataOutput cdo, Object value) { - Date dt = Converter.convertToDate(value); - Codec.DateCodec.writeDateFully(cdo, dt, getTimezone()); - } - - /** {@inheritDoc} */ - @Override - protected void encodeProto(CodecDataOutput cdo, Object value) { - Date dt = Converter.convertToDate(value); - Codec.DateCodec.writeDateProto(cdo, dt, getTimezone()); - } - - /** {@inheritDoc} */ - @Override - protected Date decodeNotNull(int flag, CodecDataInput cdi) { - LocalDate date = decodeDate(flag, cdi); - if (date == null) { - return null; - } - return new Date(date.toDate().getTime()); - } -} diff --git a/src/main/java/org/tikv/types/DecimalType.java b/src/main/java/org/tikv/types/DecimalType.java deleted file mode 100644 index 888014223c..0000000000 --- a/src/main/java/org/tikv/types/DecimalType.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import com.pingcap.tidb.tipb.ExprType; -import java.math.BigDecimal; -import org.tikv.codec.Codec; -import org.tikv.codec.Codec.DecimalCodec; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.InvalidCodecFormatException; -import org.tikv.meta.TiColumnInfo; - -public class DecimalType extends DataType { - public static final DecimalType DECIMAL = new DecimalType(MySQLType.TypeNewDecimal); - public static final MySQLType[] subTypes = new MySQLType[] {MySQLType.TypeNewDecimal}; - - private DecimalType(MySQLType tp) { - super(tp); - } - - DecimalType(TiColumnInfo.InternalTypeHolder holder) { - super(holder); - } - - /** {@inheritDoc} */ - @Override - protected Object decodeNotNull(int flag, CodecDataInput cdi) { - if (flag != Codec.DECIMAL_FLAG) { - throw new InvalidCodecFormatException("Invalid Flag type for decimal type: " + flag); - } - return DecimalCodec.readDecimal(cdi); - } - - @Override - protected void encodeKey(CodecDataOutput cdo, Object value) { - BigDecimal val = Converter.convertToBigDecimal(value); - DecimalCodec.writeDecimalFully(cdo, val); - } - - @Override - protected void encodeValue(CodecDataOutput cdo, Object value) { - BigDecimal val = Converter.convertToBigDecimal(value); - DecimalCodec.writeDecimalFully(cdo, val); - } - - @Override - protected void encodeProto(CodecDataOutput cdo, Object value) { - BigDecimal val = Converter.convertToBigDecimal(value); - DecimalCodec.writeDecimal(cdo, val); - } - - @Override - public ExprType getProtoExprType() { - return ExprType.MysqlDecimal; - } - - /** {@inheritDoc} */ - @Override - public Object getOriginDefaultValueNonNull(String value) { - return new BigDecimal(value); - } -} diff --git a/src/main/java/org/tikv/types/EnumType.java b/src/main/java/org/tikv/types/EnumType.java deleted file mode 100644 index 86fe3f249b..0000000000 --- a/src/main/java/org/tikv/types/EnumType.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.types; - -import com.pingcap.tidb.tipb.ExprType; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.UnsupportedTypeException; -import org.tikv.meta.TiColumnInfo; - -/** - * TODO: Support Enum Type EnumType class is set now only to indicate this type exists, so that we - * could throw UnsupportedTypeException when encountered. Its logic is not yet implemented. - */ -public class EnumType extends DataType { - public static final EnumType ENUM = new EnumType(MySQLType.TypeEnum); - - public static final MySQLType[] subTypes = new MySQLType[] {MySQLType.TypeEnum}; - - private EnumType(MySQLType tp) { - super(tp); - } - - protected EnumType(TiColumnInfo.InternalTypeHolder holder) { - super(holder); - } - - /** {@inheritDoc} */ - @Override - protected Object decodeNotNull(int flag, CodecDataInput cdi) { - throw new UnsupportedTypeException("Enum type not supported"); - } - - /** {@inheritDoc} Enum is encoded as unsigned int64 with its 0-based value. */ - @Override - protected void encodeKey(CodecDataOutput cdo, Object value) { - throw new UnsupportedTypeException("Enum type not supported"); - } - - /** {@inheritDoc} Enum is encoded as unsigned int64 with its 0-based value. */ - @Override - protected void encodeValue(CodecDataOutput cdo, Object value) { - throw new UnsupportedTypeException("Enum type not supported"); - } - - /** {@inheritDoc} */ - @Override - protected void encodeProto(CodecDataOutput cdo, Object value) { - throw new UnsupportedTypeException("Enum type not supported"); - } - - @Override - public ExprType getProtoExprType() { - return ExprType.MysqlEnum; - } - - /** {@inheritDoc} */ - @Override - public Object getOriginDefaultValueNonNull(String value) { - return value; - } -} diff --git a/src/main/java/org/tikv/types/IntegerType.java b/src/main/java/org/tikv/types/IntegerType.java deleted file mode 100644 index 31a78ba13b..0000000000 --- a/src/main/java/org/tikv/types/IntegerType.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import com.google.common.primitives.UnsignedLong; -import com.pingcap.tidb.tipb.ExprType; -import java.math.BigDecimal; -import org.tikv.codec.Codec; -import org.tikv.codec.Codec.IntegerCodec; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.TypeException; -import org.tikv.meta.Collation; -import org.tikv.meta.TiColumnInfo; - -public class IntegerType extends DataType { - public static final IntegerType TINYINT = new IntegerType(MySQLType.TypeTiny); - public static final IntegerType SMALLINT = new IntegerType(MySQLType.TypeShort); - public static final IntegerType MEDIUMINT = new IntegerType(MySQLType.TypeInt24); - public static final IntegerType INT = new IntegerType(MySQLType.TypeLong); - public static final IntegerType BIGINT = new IntegerType(MySQLType.TypeLonglong); - public static final IntegerType BOOLEAN = TINYINT; - - public static final IntegerType ROW_ID_TYPE = - new IntegerType(MySQLType.TypeLonglong, PriKeyFlag, 20, 0); - - public static final MySQLType[] subTypes = - new MySQLType[] { - MySQLType.TypeTiny, - MySQLType.TypeShort, - MySQLType.TypeInt24, - MySQLType.TypeLong, - MySQLType.TypeLonglong - }; - - protected IntegerType(MySQLType type, int flag, int len, int decimal) { - super(type, flag, len, decimal, "", Collation.DEF_COLLATION_CODE); - } - - protected IntegerType(MySQLType tp) { - super(tp); - } - - private static BigDecimal unsignedValueOf(long x) { - return new BigDecimal(UnsignedLong.fromLongBits(x).bigIntegerValue()); - } - - /** {@inheritDoc} */ - @Override - protected Object decodeNotNull(int flag, CodecDataInput cdi) { - long ret; - switch (flag) { - case Codec.UVARINT_FLAG: - ret = IntegerCodec.readUVarLong(cdi); - break; - case Codec.UINT_FLAG: - ret = IntegerCodec.readULong(cdi); - break; - case Codec.VARINT_FLAG: - ret = IntegerCodec.readVarLong(cdi); - break; - case Codec.INT_FLAG: - ret = IntegerCodec.readLong(cdi); - break; - default: - throw new TypeException("Invalid IntegerType flag: " + flag); - } - if (isUnsignedLong()) { - return unsignedValueOf(ret); - } - return ret; - } - - /** {@inheritDoc} */ - @Override - protected void encodeKey(CodecDataOutput cdo, Object value) { - long longVal = Converter.convertToLong(value); - if (isUnsigned()) { - IntegerCodec.writeULongFully(cdo, longVal, true); - } else { - IntegerCodec.writeLongFully(cdo, longVal, true); - } - } - - /** {@inheritDoc} */ - @Override - protected void encodeValue(CodecDataOutput cdo, Object value) { - long longVal = Converter.convertToLong(value); - if (isUnsigned()) { - IntegerCodec.writeULongFully(cdo, longVal, false); - } else { - IntegerCodec.writeLongFully(cdo, longVal, false); - } - } - - /** {@inheritDoc} */ - @Override - protected void encodeProto(CodecDataOutput cdo, Object value) { - long longVal = Converter.convertToLong(value); - if (isUnsigned()) { - IntegerCodec.writeULong(cdo, longVal); - } else { - IntegerCodec.writeLong(cdo, longVal); - } - } - - @Override - public ExprType getProtoExprType() { - return isUnsigned() ? ExprType.Uint64 : ExprType.Int64; - } - - public boolean isUnsignedLong() { - return tp == MySQLType.TypeLonglong && isUnsigned(); - } - - public boolean isUnsigned() { - return (flag & UnsignedFlag) > 0; - } - /** {@inheritDoc} */ - @Override - public Object getOriginDefaultValueNonNull(String value) { - return Integer.parseInt(value); - } - - protected IntegerType(TiColumnInfo.InternalTypeHolder holder) { - super(holder); - } -} diff --git a/src/main/java/org/tikv/types/JsonType.java b/src/main/java/org/tikv/types/JsonType.java deleted file mode 100644 index 7175d0306a..0000000000 --- a/src/main/java/org/tikv/types/JsonType.java +++ /dev/null @@ -1,374 +0,0 @@ -package org.tikv.types; - -import com.google.common.io.ByteArrayDataInput; -import com.google.common.io.ByteStreams; -import com.google.gson.*; -import com.pingcap.tidb.tipb.ExprType; -import java.io.DataInput; -import java.io.IOException; -import java.io.UncheckedIOException; -import javax.annotation.Nullable; -import org.apache.commons.io.Charsets; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.meta.TiColumnInfo; - -public class JsonType extends DataType { - - private static final int KEY_ENTRY_LENGTH = 6; - private static final int VALUE_ENTRY_SIZE = 5; - - // TypeCodeObject indicates the JSON is an object. - private static final byte TYPE_CODE_OBJECT = 0x01; - // TypeCodeArray indicates the JSON is an array. - private static final byte TYPE_CODE_ARRAY = 0x03; - // TypeCodeLiteral indicates the JSON is a literal. - private static final byte TYPE_CODE_LITERAL = 0x04; - // TypeCodeInt64 indicates the JSON is a signed integer. - private static final byte TYPE_CODE_INT64 = 0x09; - // TypeCodeUint64 indicates the JSON is a unsigned integer. - private static final byte TYPE_CODE_UINT64 = 0x0a; - // TypeCodeFloat64 indicates the JSON is a double float number. - private static final byte TYPE_CODE_FLOAT64 = 0x0b; - // TypeCodeString indicates the JSON is a string. - private static final byte TYPE_CODE_STRING = 0x0c; - - // LiteralNil represents JSON null. - private static final byte LITERAL_NIL = 0x00; - // LiteralTrue represents JSON true. - private static final byte LITERAL_TRUE = 0x01; - // LiteralFalse represents JSON false. - private static final byte LITERAL_FALSE = 0x02; - private static final JsonPrimitive JSON_FALSE = new JsonPrimitive(false); - private static final JsonPrimitive JSON_TRUE = new JsonPrimitive(true); - public static MySQLType[] subTypes = new MySQLType[] {MySQLType.TypeJSON}; - - protected JsonType(TiColumnInfo.InternalTypeHolder holder) { - super(holder); - } - - public JsonType(MySQLType type) { - super(type); - } - - public JsonType(MySQLType type, int flag, int len, int decimal, String charset, int collation) { - super(type, flag, len, decimal, charset, collation); - } - - @Override - protected Object decodeNotNull(int flag, CodecDataInput cdi) { - byte type = readByte(cdi); - return parseValue(type, cdi).toString(); - } - - /* - The binary JSON format from MySQL 5.7 is as follows: - JSON doc ::= type value - type ::= - 0x01 | // large JSON object - 0x03 | // large JSON array - 0x04 | // literal (true/false/null) - 0x05 | // int16 - 0x06 | // uint16 - 0x07 | // int32 - 0x08 | // uint32 - 0x09 | // int64 - 0x0a | // uint64 - 0x0b | // double - 0x0c | // utf8mb4 string - value ::= - object | - array | - literal | - number | - string | - object ::= element-count size key-entry* value-entry* key* value* - array ::= element-count size value-entry* value* - // number of members in object or number of elements in array - element-count ::= uint32 - // number of bytes in the binary representation of the object or array - size ::= uint32 - key-entry ::= key-offset key-length - key-offset ::= uint32 - key-length ::= uint16 // key length must be less than 64KB - value-entry ::= type offset-or-inlined-value - // This field holds either the offset to where the value is stored, - // or the value itself if it is small enough to be inlined (that is, - // if it is a JSON literal or a small enough [u]int). - offset-or-inlined-value ::= uint32 - key ::= utf8mb4-data - literal ::= - 0x00 | // JSON null literal - 0x01 | // JSON true literal - 0x02 | // JSON false literal - number ::= .... // little-endian format for [u]int(16|32|64), whereas - // double is stored in a platform-independent, eight-byte - // format using float8store() - string ::= data-length utf8mb4-data - data-length ::= uint8* // If the high bit of a byte is 1, the length - // field is continued in the next byte, - // otherwise it is the last byte of the length - // field. So we need 1 byte to represent - // lengths up to 127, 2 bytes to represent - // lengths up to 16383, and so on... - */ - private JsonElement parseValue(byte type, DataInput di) { - switch (type) { - case TYPE_CODE_OBJECT: - return parseObject(di); - case TYPE_CODE_ARRAY: - return parseArray(di); - case TYPE_CODE_LITERAL: - return parseLiteralJson(di); - case TYPE_CODE_INT64: - return new JsonPrimitive(parseInt64(di)); - case TYPE_CODE_UINT64: - return new JsonPrimitive(parseUint64(di)); - case TYPE_CODE_FLOAT64: - return new JsonPrimitive(parseDouble(di)); - case TYPE_CODE_STRING: - long length = parseDataLength(di); - return new JsonPrimitive(parseString(di, length)); - default: - throw new AssertionError("error type|type=" + (int) type); - } - } - - // * notice use this as a unsigned long - private long parseUint64(DataInput cdi) { - byte[] readBuffer = new byte[8]; - readFully(cdi, readBuffer, 0, 8); - - return ((long) (readBuffer[7]) << 56) - + ((long) (readBuffer[6] & 255) << 48) - + ((long) (readBuffer[5] & 255) << 40) - + ((long) (readBuffer[4] & 255) << 32) - + ((long) (readBuffer[3] & 255) << 24) - + ((readBuffer[2] & 255) << 16) - + ((readBuffer[1] & 255) << 8) - + ((readBuffer[0] & 255) << 0); - } - - private void readFully(DataInput cdi, byte[] readBuffer, final int off, final int len) { - try { - cdi.readFully(readBuffer, off, len); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private long parseInt64(DataInput cdi) { - byte[] readBuffer = new byte[8]; - readFully(cdi, readBuffer); - return ((long) readBuffer[7] << 56) - + ((long) (readBuffer[6] & 255) << 48) - + ((long) (readBuffer[5] & 255) << 40) - + ((long) (readBuffer[4] & 255) << 32) - + ((long) (readBuffer[3] & 255) << 24) - + ((readBuffer[2] & 255) << 16) - + ((readBuffer[1] & 255) << 8) - + ((readBuffer[0] & 255) << 0); - } - - private long parseUint32(DataInput cdi) { - byte[] readBuffer = new byte[4]; - readFully(cdi, readBuffer); - - return ((long) (readBuffer[3] & 255) << 24) - + ((readBuffer[2] & 255) << 16) - + ((readBuffer[1] & 255) << 8) - + ((readBuffer[0] & 255) << 0); - } - - private double parseDouble(DataInput cdi) { - byte[] readBuffer = new byte[8]; - readFully(cdi, readBuffer); - return Double.longBitsToDouble( - ((long) readBuffer[7] << 56) - + ((long) (readBuffer[6] & 255) << 48) - + ((long) (readBuffer[5] & 255) << 40) - + ((long) (readBuffer[4] & 255) << 32) - + ((long) (readBuffer[3] & 255) << 24) - + ((readBuffer[2] & 255) << 16) - + ((readBuffer[1] & 255) << 8) - + ((readBuffer[0] & 255) << 0)); - } - - private int parseUint16(DataInput cdi) { - byte[] readBuffer = new byte[2]; - readFully(cdi, readBuffer); - - return ((readBuffer[1] & 255) << 8) + ((readBuffer[0] & 255) << 0); - } - - private String parseString(DataInput di, long length) { - - byte[] buffer = new byte[Math.toIntExact(length)]; - readFully(di, buffer); - return new String(buffer, Charsets.UTF_8); - } - - /** - * func Uvarint(buf []byte) (uint64, int) { var x uint64 var s uint for i, b := range buf { if b < - * 0x80 { if i > 9 || i == 9 && b > 1 { return 0, -(i + 1) // overflow } return x | uint64(b)<=9 leading bytes"); - } - x |= ((long) (b & 0x7f)) << s; - s += 7; - i++; - } - - if (i == 9 && b > 1) { - throw new IllegalArgumentException("overflow: 8 leading byte and last one > 1"); - } - x |= ((long) b) << s; - return x; - } - - private @Nullable Boolean parseLiteral(DataInput cdi) { - byte type; - type = readByte(cdi); - switch (type) { - case LITERAL_FALSE: - return Boolean.FALSE; - case LITERAL_NIL: - return null; - case LITERAL_TRUE: - return Boolean.TRUE; - default: - throw new AssertionError("unknown literal type|" + (int) type); - } - } - - private byte readByte(DataInput cdi) { - byte type; - try { - type = cdi.readByte(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return type; - } - - private JsonArray parseArray(DataInput di) { - long elementCount = parseUint32(di); - long size = parseUint32(di); - byte[] buffer = new byte[Math.toIntExact(size - 8)]; - readFully(di, buffer); - JsonArray jsonArray = new JsonArray(); - for (int i = 0; i < elementCount; i++) { - JsonElement value = parseValueEntry(buffer, VALUE_ENTRY_SIZE * i); - jsonArray.add(value); - } - return jsonArray; - } - - private JsonObject parseObject(DataInput di) { - long elementCount = parseUint32(di); - long size = parseUint32(di); - - byte[] buffer = new byte[Math.toIntExact(size - 8)]; - readFully(di, buffer); - JsonObject jsonObject = new JsonObject(); - for (int i = 0; i < elementCount; i++) { - KeyEntry keyEntry = parseKeyEntry(ByteStreams.newDataInput(buffer, i * KEY_ENTRY_LENGTH)); - String key = - parseString( - ByteStreams.newDataInput(buffer, Math.toIntExact(keyEntry.keyOffset - 8)), - keyEntry.keyLength); - long valueEntryOffset = elementCount * KEY_ENTRY_LENGTH + i * VALUE_ENTRY_SIZE; - JsonElement value = parseValueEntry(buffer, valueEntryOffset); - jsonObject.add(key, value); - } - return jsonObject; - } - - private JsonElement parseValueEntry(byte[] buffer, long valueEntryOffset) { - byte valueType = buffer[Math.toIntExact(valueEntryOffset)]; - JsonElement value; - ByteArrayDataInput bs = ByteStreams.newDataInput(buffer, Math.toIntExact(valueEntryOffset + 1)); - switch (valueType) { - case TYPE_CODE_LITERAL: - value = parseLiteralJson(bs); - break; - default: - long valueOffset = parseUint32(bs); - value = - parseValue( - valueType, ByteStreams.newDataInput(buffer, Math.toIntExact(valueOffset - 8))); - } - return value; - } - - private JsonElement parseLiteralJson(DataInput di) { - JsonElement value; - Boolean bool = parseLiteral(di); - if (bool == null) { - value = JsonNull.INSTANCE; - } else if (bool) { - value = JSON_TRUE; - } else { - value = JSON_FALSE; - } - return value; - } - - private KeyEntry parseKeyEntry(DataInput di) { - return new KeyEntry(parseUint32(di), parseUint16(di)); - } - - static class KeyEntry { - long keyOffset; - int keyLength; - - public KeyEntry(long keyOffset, int keyLength) { - this.keyOffset = keyOffset; - this.keyLength = keyLength; - } - } - - private void readFully(DataInput di, byte[] buffer) { - try { - di.readFully(buffer); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - @Override - protected void encodeKey(CodecDataOutput cdo, Object value) { - throw new UnsupportedOperationException("JsonType.encodeKey|value=" + value); - } - - @Override - protected void encodeValue(CodecDataOutput cdo, Object value) { - throw new UnsupportedOperationException("JsonType.encodeValue|value=" + value); - } - - @Override - protected void encodeProto(CodecDataOutput cdo, Object value) { - throw new UnsupportedOperationException("JsonType.encodeProto|value=" + value); - } - - @Override - public ExprType getProtoExprType() { - return ExprType.MysqlJson; - } - - @Override - public Object getOriginDefaultValueNonNull(String value) { - throw new AssertionError("json can't have a default value"); - } -} diff --git a/src/main/java/org/tikv/types/MySQLType.java b/src/main/java/org/tikv/types/MySQLType.java deleted file mode 100644 index e0563064ee..0000000000 --- a/src/main/java/org/tikv/types/MySQLType.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import static java.util.Objects.requireNonNull; - -import java.util.HashMap; -import java.util.Map; - -public enum MySQLType { - TypeDecimal(0), - TypeTiny(1), - TypeShort(2), - TypeLong(3), - TypeFloat(4), - TypeDouble(5), - TypeNull(6), - TypeTimestamp(7), - TypeLonglong(8), - TypeInt24(9), - TypeDate(10), - TypeDuration(11), - TypeDatetime(12), - TypeYear(13), - TypeNewDate(14), - TypeVarchar(15), - TypeBit(16), - TypeJSON(0xf5), - TypeNewDecimal(0xf6), - TypeEnum(0xf7), - TypeSet(0xf8), - TypeTinyBlob(0xf9), - TypeMediumBlob(0xfa), - TypeLongBlob(0xfb), - TypeBlob(0xfc), - TypeVarString(0xfd), - TypeString(0xfe), - TypeGeometry(0xff); - - private static final Map typeMap = new HashMap<>(); - - static { - for (MySQLType type : MySQLType.values()) { - typeMap.put(type.getTypeCode(), type); - } - } - - private int typeCode; - - MySQLType(int tp) { - typeCode = tp; - } - - public int getTypeCode() { - return typeCode; - } - - public static MySQLType fromTypeCode(int typeCode) { - MySQLType type = typeMap.get(typeCode); - return requireNonNull(type, String.format("Cannot find Type from type code %d", typeCode)); - } -} diff --git a/src/main/java/org/tikv/types/RealType.java b/src/main/java/org/tikv/types/RealType.java deleted file mode 100644 index 83d304a65d..0000000000 --- a/src/main/java/org/tikv/types/RealType.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import com.pingcap.tidb.tipb.ExprType; -import org.tikv.codec.Codec; -import org.tikv.codec.Codec.DecimalCodec; -import org.tikv.codec.Codec.RealCodec; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.InvalidCodecFormatException; -import org.tikv.exception.TypeException; -import org.tikv.meta.TiColumnInfo; - -public class RealType extends DataType { - public static final RealType DOUBLE = new RealType(MySQLType.TypeDouble); - public static final RealType FLOAT = new RealType(MySQLType.TypeFloat); - public static final RealType REAL = DOUBLE; - - public static final MySQLType[] subTypes = - new MySQLType[] {MySQLType.TypeDouble, MySQLType.TypeFloat}; - - private RealType(MySQLType tp) { - super(tp); - } - - RealType(TiColumnInfo.InternalTypeHolder holder) { - super(holder); - } - - /** {@inheritDoc} */ - @Override - protected Object decodeNotNull(int flag, CodecDataInput cdi) { - if (flag == Codec.DECIMAL_FLAG) { - return DecimalCodec.readDecimal(cdi).doubleValue(); - } else if (flag == Codec.FLOATING_FLAG) { - return RealCodec.readDouble(cdi); - } - throw new InvalidCodecFormatException("Invalid Flag type for float type: " + flag); - } - - @Override - protected void encodeKey(CodecDataOutput cdo, Object value) { - double val = Converter.convertToDouble(value); - RealCodec.writeDoubleFully(cdo, val); - } - - @Override - protected void encodeValue(CodecDataOutput cdo, Object value) { - double val = Converter.convertToDouble(value); - RealCodec.writeDoubleFully(cdo, val); - } - - @Override - protected void encodeProto(CodecDataOutput cdo, Object value) { - double val = Converter.convertToDouble(value); - RealCodec.writeDouble(cdo, val); - } - - @Override - public ExprType getProtoExprType() { - if (tp == MySQLType.TypeDouble) { - return ExprType.Float64; - } else if (tp == MySQLType.TypeFloat) { - return ExprType.Float32; - } - throw new TypeException("Unknown Type encoding proto " + tp); - } - - /** {@inheritDoc} */ - @Override - public Object getOriginDefaultValueNonNull(String value) { - return Double.parseDouble(value); - } -} diff --git a/src/main/java/org/tikv/types/SetType.java b/src/main/java/org/tikv/types/SetType.java deleted file mode 100644 index 63ee91b5af..0000000000 --- a/src/main/java/org/tikv/types/SetType.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.types; - -import com.pingcap.tidb.tipb.ExprType; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.UnsupportedTypeException; -import org.tikv.meta.TiColumnInfo; - -/** - * TODO: Support Set Type SetType class is set now only to indicate this type exists, so that we - * could throw UnsupportedTypeException when encountered. Its logic is not yet implemented. - * - *

Set is encoded as unsigned int64 with its 0-based value. - */ -public class SetType extends DataType { - public static final SetType SET = new SetType(MySQLType.TypeSet); - - public static final MySQLType[] subTypes = new MySQLType[] {MySQLType.TypeSet}; - - private SetType(MySQLType tp) { - super(tp); - } - - protected SetType(TiColumnInfo.InternalTypeHolder holder) { - super(holder); - } - - /** {@inheritDoc} */ - @Override - protected Object decodeNotNull(int flag, CodecDataInput cdi) { - throw new UnsupportedTypeException("Set type not supported"); - } - - /** {@inheritDoc} */ - @Override - protected void encodeKey(CodecDataOutput cdo, Object value) { - throw new UnsupportedTypeException("Set type not supported"); - } - - /** {@inheritDoc} */ - @Override - protected void encodeValue(CodecDataOutput cdo, Object value) { - throw new UnsupportedTypeException("Set type not supported"); - } - - /** {@inheritDoc} */ - @Override - protected void encodeProto(CodecDataOutput cdo, Object value) { - throw new UnsupportedTypeException("Set type not supported"); - } - - @Override - public ExprType getProtoExprType() { - return ExprType.MysqlSet; - } - - /** {@inheritDoc} */ - @Override - public Object getOriginDefaultValueNonNull(String value) { - return value; - } -} diff --git a/src/main/java/org/tikv/types/StringType.java b/src/main/java/org/tikv/types/StringType.java deleted file mode 100644 index 322dcb5483..0000000000 --- a/src/main/java/org/tikv/types/StringType.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import java.nio.charset.StandardCharsets; -import org.tikv.codec.CodecDataInput; -import org.tikv.meta.TiColumnInfo; - -public class StringType extends BytesType { - public static final StringType VARCHAR = new StringType(MySQLType.TypeVarchar); - public static final StringType BINARY = new StringType(MySQLType.TypeString); - public static final StringType CHAR = new StringType(MySQLType.TypeString); - - public static final MySQLType[] subTypes = - new MySQLType[] {MySQLType.TypeVarchar, MySQLType.TypeString}; - - protected StringType(MySQLType tp) { - super(tp); - } - - protected StringType(TiColumnInfo.InternalTypeHolder holder) { - super(holder); - } - - /** {@inheritDoc} */ - @Override - protected Object decodeNotNull(int flag, CodecDataInput cdi) { - return new String((byte[]) super.decodeNotNull(flag, cdi), StandardCharsets.UTF_8); - } -} diff --git a/src/main/java/org/tikv/types/TimestampType.java b/src/main/java/org/tikv/types/TimestampType.java deleted file mode 100644 index 0fe9773ec2..0000000000 --- a/src/main/java/org/tikv/types/TimestampType.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import java.sql.Timestamp; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.tikv.codec.Codec.DateTimeCodec; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.meta.TiColumnInfo; - -/** - * Timestamp in TiDB is represented as packed long including year/month and etc. When stored, it is - * converted into UTC and when read it should be interpreted as UTC and convert to local When - * encoded as coprocessor request 1. all date time in key should be converted to UTC 2. all date - * time in proto should be converted into local timezone 3. all incoming data should be regarded as - * local timezone if no timezone indicated - * - *

For example, Encoding: If incoming literal is a string '2000-01-01 00:00:00' which has no - * timezone, it interpreted as text in local timezone and encoded with UTC; If incoming literal is a - * epoch millisecond, it interpreted as UTC epoch and encode with UTC if index / key or local - * timezone if proto - */ -public class TimestampType extends AbstractDateTimeType { - public static final TimestampType TIMESTAMP = new TimestampType(MySQLType.TypeTimestamp); - public static final TimestampType TIME = new TimestampType(MySQLType.TypeDuration); - - public static final MySQLType[] subTypes = - new MySQLType[] {MySQLType.TypeTimestamp, MySQLType.TypeDuration}; - - TimestampType(MySQLType tp) { - super(tp); - } - - TimestampType(TiColumnInfo.InternalTypeHolder holder) { - super(holder); - } - - protected DateTimeZone getTimezone() { - return DateTimeZone.UTC; - } - - /** - * Decode timestamp from packed long value In TiDB / MySQL, timestamp type is converted to UTC and - * stored - */ - @Override - protected Timestamp decodeNotNull(int flag, CodecDataInput cdi) { - DateTime dateTime = decodeDateTime(flag, cdi); - if (dateTime == null) { - return null; - } - return new Timestamp(dateTime.getMillis()); - } - - @Override - public DateTime getOriginDefaultValueNonNull(String value) { - return Converter.convertToDateTime(value); - } - - /** {@inheritDoc} */ - @Override - protected void encodeProto(CodecDataOutput cdo, Object value) { - DateTime dt = Converter.convertToDateTime(value); - DateTimeCodec.writeDateTimeProto(cdo, dt, Converter.getLocalTimezone()); - } -} diff --git a/src/main/java/org/tikv/types/YearType.java b/src/main/java/org/tikv/types/YearType.java deleted file mode 100644 index 098bb911a7..0000000000 --- a/src/main/java/org/tikv/types/YearType.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.types; - -import com.pingcap.tidb.tipb.ExprType; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.UnsupportedTypeException; -import org.tikv.meta.TiColumnInfo; - -/** - * TODO: Support Year Type YearType class is set now only to indicate this type exists, so that we - * could throw UnsupportedTypeException when encountered. Its logic is not yet implemented. - * - *

Since year type acts differently in Spark and MySQL -- for instance, in MySQL, year is an - * unsigned integer(2017), whereas in Spark, year is treated as Date(2017-01-01). -- it is not - * decided which logic should inherit. - * - *

Year is encoded as unsigned int64. - */ -public class YearType extends DataType { - public static final YearType YEAR = new YearType(MySQLType.TypeYear); - - public static final MySQLType[] subTypes = new MySQLType[] {MySQLType.TypeYear}; - - private YearType(MySQLType tp) { - super(tp); - } - - protected YearType(TiColumnInfo.InternalTypeHolder holder) { - super(holder); - } - - /** {@inheritDoc} */ - @Override - protected Object decodeNotNull(int flag, CodecDataInput cdi) { - throw new UnsupportedTypeException("Year type not supported"); - } - - /** {@inheritDoc} */ - @Override - protected void encodeKey(CodecDataOutput cdo, Object value) { - throw new UnsupportedTypeException("Year type not supported"); - } - - /** {@inheritDoc} */ - @Override - protected void encodeValue(CodecDataOutput cdo, Object value) { - throw new UnsupportedTypeException("Year type not supported"); - } - - /** {@inheritDoc} */ - @Override - protected void encodeProto(CodecDataOutput cdo, Object value) { - throw new UnsupportedTypeException("Year type not supported"); - } - - @Override - public ExprType getProtoExprType() { - return ExprType.Int64; - } - - /** {@inheritDoc} */ - @Override - public Object getOriginDefaultValueNonNull(String value) { - return value; - } -} diff --git a/src/main/java/org/tikv/util/BackOffFunction.java b/src/main/java/org/tikv/util/BackOffFunction.java index e10cfdf9eb..c99aa08355 100644 --- a/src/main/java/org/tikv/util/BackOffFunction.java +++ b/src/main/java/org/tikv/util/BackOffFunction.java @@ -28,7 +28,8 @@ public class BackOffFunction { BoPDRPC, BoRegionMiss, BoUpdateLeader, - BoServerBusy + BoServerBusy, + BoStoreNotMatch } /** diff --git a/src/main/java/org/tikv/util/ConcreteBackOffer.java b/src/main/java/org/tikv/util/ConcreteBackOffer.java index fa70a0e052..0fbc5b76aa 100644 --- a/src/main/java/org/tikv/util/ConcreteBackOffer.java +++ b/src/main/java/org/tikv/util/ConcreteBackOffer.java @@ -95,6 +95,9 @@ public class ConcreteBackOffer implements BackOffer { case BoTiKVRPC: backOffFunction = BackOffFunction.create(500, 5000, BackOffStrategy.EqualJitter); break; + case BoStoreNotMatch: + backOffFunction = BackOffFunction.create(100, 3000, BackOffStrategy.EqualJitter); + break; } return backOffFunction; } diff --git a/src/main/java/org/tikv/util/RangeSplitter.java b/src/main/java/org/tikv/util/RangeSplitter.java deleted file mode 100644 index 83e9f65b94..0000000000 --- a/src/main/java/org/tikv/util/RangeSplitter.java +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.util; - -import static org.tikv.key.Key.toRawKey; -import static org.tikv.util.KeyRangeUtils.formatByteString; -import static org.tikv.util.KeyRangeUtils.makeCoprocRange; - -import com.google.common.collect.ImmutableList; -import com.google.common.net.HostAndPort; -import com.google.protobuf.ByteString; -import gnu.trove.list.array.TLongArrayList; -import gnu.trove.map.hash.TLongObjectHashMap; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.tikv.exception.TiExpressionException; -import org.tikv.key.Key; -import org.tikv.key.RowKey; -import org.tikv.key.RowKey.DecodeResult; -import org.tikv.key.RowKey.DecodeResult.Status; -import org.tikv.kvproto.Coprocessor.KeyRange; -import org.tikv.kvproto.Metapb; -import org.tikv.region.RegionManager; -import org.tikv.region.TiRegion; - -public class RangeSplitter { - public static class RegionTask implements Serializable { - private final TiRegion region; - private final Metapb.Store store; - private final List ranges; - private final String host; - - public static RegionTask newInstance( - TiRegion region, Metapb.Store store, List ranges) { - return new RegionTask(region, store, ranges); - } - - RegionTask(TiRegion region, Metapb.Store store, List ranges) { - this.region = region; - this.store = store; - this.ranges = ranges; - String host = null; - try { - host = HostAndPort.fromString(store.getAddress()).getHostText(); - } catch (Exception ignored) { - } - this.host = host; - } - - public TiRegion getRegion() { - return region; - } - - public Metapb.Store getStore() { - return store; - } - - public List getRanges() { - return ranges; - } - - public String getHost() { - return host; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(String.format("Region [%s]", region)); - sb.append(" "); - - for (KeyRange range : ranges) { - sb.append( - String.format( - "Range Start: [%s] Range End: [%s]", - formatByteString(range.getStart()), formatByteString(range.getEnd()))); - } - - return sb.toString(); - } - } - - public static RangeSplitter newSplitter(RegionManager mgr) { - return new RangeSplitter(mgr); - } - - private RangeSplitter(RegionManager regionManager) { - this.regionManager = regionManager; - } - - private final RegionManager regionManager; - - /** - * Group by a list of handles by the handles' region id, handles will be sorted. - * - * @param tableId Table id used for the handle - * @param handles Handle list - * @return map - */ - public TLongObjectHashMap groupByAndSortHandlesByRegionId( - long tableId, TLongArrayList handles) { - TLongObjectHashMap result = new TLongObjectHashMap<>(); - handles.sort(); - - int startPos = 0; - DecodeResult decodeResult = new DecodeResult(); - while (startPos < handles.size()) { - long curHandle = handles.get(startPos); - RowKey key = RowKey.toRowKey(tableId, curHandle); - Pair regionStorePair = - regionManager.getRegionStorePairByKey(ByteString.copyFrom(key.getBytes())); - byte[] endKey = regionStorePair.first.getEndKey().toByteArray(); - RowKey.tryDecodeRowKey(tableId, endKey, decodeResult); - if (decodeResult.status == Status.MIN) { - throw new TiExpressionException("EndKey is less than current rowKey"); - } else if (decodeResult.status == Status.MAX || decodeResult.status == Status.UNKNOWN_INF) { - result.put( - regionStorePair.first.getId(), createHandleList(startPos, handles.size(), handles)); - break; - } - - // Region range is a close-open range - // If region end key match exactly or slightly less than a handle, - // that handle should be excluded from current region - // If region end key is greater than the handle, that handle should be included - long regionEndHandle = decodeResult.handle; - int pos = handles.binarySearch(regionEndHandle, startPos, handles.size()); - - if (pos < 0) { - // not found in handles, pos is the next greater pos - // [startPos, pos) all included - pos = -(pos + 1); - } else if (decodeResult.status == Status.GREATER) { - // found handle and then further consider decode status - // End key decode to a value v: regionEndHandle < v < regionEndHandle + 1 - // handle at pos included - pos++; - } - result.put(regionStorePair.first.getId(), createHandleList(startPos, pos, handles)); - // pos equals to start leads to an dead loop - // startPos and its handle is used for searching region in PD. - // The returning close-open range should at least include startPos's handle - // so only if PD error and startPos is not included in current region then startPos == pos - if (startPos >= pos) { - throw new TiExpressionException("searchKey is not included in region returned by PD"); - } - startPos = pos; - } - - return result; - } - - private TLongArrayList createHandleList(int startPos, int endPos, TLongArrayList handles) { - TLongArrayList result = new TLongArrayList(); - for (int i = startPos; i < endPos; i++) { - result.add(handles.get(i)); - } - return result; - } - - /** - * Build region tasks from handles split by region, handles will be sorted. - * - * @param tableId Table ID - * @param handles Handle list - * @return A list of region tasks - */ - public List splitAndSortHandlesByRegion(long tableId, TLongArrayList handles) { - // Max value for current index handle range - ImmutableList.Builder regionTasks = ImmutableList.builder(); - - TLongObjectHashMap regionHandlesMap = - groupByAndSortHandlesByRegionId(tableId, handles); - - regionHandlesMap.forEachEntry( - (k, v) -> { - Pair regionStorePair = - regionManager.getRegionStorePairByRegionId(k); - createTask(0, v.size(), tableId, v, regionStorePair, regionTasks); - return true; - }); - - return regionTasks.build(); - } - - private void createTask( - int startPos, - int endPos, - long tableId, - TLongArrayList handles, - Pair regionStorePair, - ImmutableList.Builder regionTasks) { - List newKeyRanges = new ArrayList<>(endPos - startPos + 1); - long startHandle = handles.get(startPos); - long endHandle = startHandle; - for (int i = startPos + 1; i < endPos; i++) { - long curHandle = handles.get(i); - if (endHandle + 1 == curHandle) { - endHandle = curHandle; - } else { - newKeyRanges.add( - makeCoprocRange( - RowKey.toRowKey(tableId, startHandle).toByteString(), - RowKey.toRowKey(tableId, endHandle + 1).toByteString())); - startHandle = curHandle; - endHandle = startHandle; - } - } - newKeyRanges.add( - makeCoprocRange( - RowKey.toRowKey(tableId, startHandle).toByteString(), - RowKey.toRowKey(tableId, endHandle + 1).toByteString())); - regionTasks.add(new RegionTask(regionStorePair.first, regionStorePair.second, newKeyRanges)); - } - - /** - * Split key ranges into corresponding region tasks and group by their region id - * - * @param keyRanges List of key ranges - * @return List of RegionTask, each task corresponds to a different region. - */ - public List splitRangeByRegion(List keyRanges) { - if (keyRanges == null || keyRanges.size() == 0) { - return ImmutableList.of(); - } - - int i = 0; - KeyRange range = keyRanges.get(i++); - Map> idToRange = new HashMap<>(); // region id to keyRange list - Map> idToRegion = new HashMap<>(); - - while (true) { - Pair regionStorePair = - regionManager.getRegionStorePairByKey(range.getStart()); - - if (regionStorePair == null) { - throw new NullPointerException( - "fail to get region/store pair by key " + formatByteString(range.getStart())); - } - TiRegion region = regionStorePair.first; - idToRegion.putIfAbsent(region.getId(), regionStorePair); - - // both key range is close-opened - // initial range inside PD is guaranteed to be -INF to +INF - // Both keys are at right hand side and then always not -INF - if (toRawKey(range.getEnd()).compareTo(toRawKey(region.getEndKey())) > 0) { - // current region does not cover current end key - KeyRange cutRange = - KeyRange.newBuilder().setStart(range.getStart()).setEnd(region.getEndKey()).build(); - - List ranges = idToRange.computeIfAbsent(region.getId(), k -> new ArrayList<>()); - ranges.add(cutRange); - - // cut new remaining for current range - range = KeyRange.newBuilder().setStart(region.getEndKey()).setEnd(range.getEnd()).build(); - } else { - // current range covered by region - List ranges = idToRange.computeIfAbsent(region.getId(), k -> new ArrayList<>()); - ranges.add(range); - if (i >= keyRanges.size()) { - break; - } - range = keyRanges.get(i++); - } - } - - ImmutableList.Builder resultBuilder = ImmutableList.builder(); - for (Map.Entry> entry : idToRange.entrySet()) { - Pair regionStorePair = idToRegion.get(entry.getKey()); - resultBuilder.add( - new RegionTask(regionStorePair.first, regionStorePair.second, entry.getValue())); - } - return resultBuilder.build(); - } - - /** - * Split SORTED key ranges with same region id into corresponding region tasks - * - * @param keyRanges List of key ranges - * @param splitNum upper bound of number of ranges to merge into - * @return List of RegionTask, each task corresponds to a different region. - */ - public List splitSortedRangeInRegion(List keyRanges, int splitNum) { - - List mergedKeyRanges = KeyRangeUtils.mergeSortedRanges(keyRanges, splitNum); - - if (mergedKeyRanges == null || mergedKeyRanges.size() == 0) { - return ImmutableList.of(); - } - - if (mergedKeyRanges.size() > splitNum) { - throw new RuntimeException("Sorted ranges were merged into more key ranges than splitNum"); - } - - int i = 0; - KeyRange range = mergedKeyRanges.get(i++); - Pair baseRegionStorePair = - regionManager.getRegionStorePairByKey(range.getStart()); - TiRegion region = baseRegionStorePair.first; - Key regionEndKey = toRawKey(region.getEndKey()); - long regionId = region.getId(); - ImmutableList.Builder resultBuilder = ImmutableList.builder(); - ImmutableList.Builder ranges = ImmutableList.builder(); - - while (true) { - Pair startKeyRegionStorePair = - regionManager.getRegionStorePairByKey(range.getStart()); - - if (startKeyRegionStorePair == null) { - throw new NullPointerException( - "fail to get region/store pair by key " + formatByteString(range.getStart())); - } - if (startKeyRegionStorePair.first.getId() != regionId) { - throw new RuntimeException("Something went wrong: key range not in current region"); - } - // both key range is close-opened - // initial range inside PD is guaranteed to be -INF to +INF - // Both keys are at right hand side and then always not -INF - if (toRawKey(range.getEnd()).compareTo(regionEndKey) > 0) { - throw new RuntimeException("Something went wrong: key range not in current region"); - } - ranges.add(range); - if (i >= mergedKeyRanges.size()) { - break; - } - range = mergedKeyRanges.get(i++); - } - - // since all ranges are guaranteed to be in same region, - // current range is covered by region - resultBuilder.add( - new RegionTask(baseRegionStorePair.first, baseRegionStorePair.second, ranges.build())); - - return resultBuilder.build(); - } -} diff --git a/src/test/java/org/tikv/RegionStoreClientTest.java b/src/test/java/org/tikv/RegionStoreClientTest.java index adedbdc155..6b4c38f720 100644 --- a/src/test/java/org/tikv/RegionStoreClientTest.java +++ b/src/test/java/org/tikv/RegionStoreClientTest.java @@ -18,14 +18,10 @@ package org.tikv; import static org.junit.Assert.*; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import com.google.protobuf.ByteString; import com.pingcap.tidb.tipb.*; import java.util.*; -import java.util.stream.Collectors; import org.junit.Test; -import org.tikv.kvproto.Coprocessor; -import org.tikv.kvproto.Coprocessor.KeyRange; import org.tikv.kvproto.Kvrpcpb; import org.tikv.kvproto.Metapb; import org.tikv.region.RegionStoreClient; @@ -146,74 +142,6 @@ public class RegionStoreClientTest extends MockServerTest { client.close(); } - @Test - public void coprocessTest() throws Exception { - RegionStoreClient client = createClient(); - - server.put("key1", "value1"); - server.put("key2", "value2"); - server.put("key4", "value4"); - server.put("key5", "value5"); - server.put("key6", "value6"); - server.put("key7", "value7"); - DAGRequest.Builder builder = DAGRequest.newBuilder(); - builder.setStartTs(1); - builder.addExecutors(Executor.newBuilder().setTp(ExecType.TypeTableScan).build()); - List keyRanges = - ImmutableList.of( - createByteStringRange(ByteString.copyFromUtf8("key1"), ByteString.copyFromUtf8("key4")), - createByteStringRange( - ByteString.copyFromUtf8("key6"), ByteString.copyFromUtf8("key7"))); - - SelectResponse resp = coprocess(client, builder.build(), keyRanges); - assertEquals(5, resp.getChunksCount()); - Set results = - ImmutableSet.copyOf( - resp.getChunksList() - .stream() - .map(c -> c.getRowsData().toStringUtf8()) - .collect(Collectors.toList())); - assertTrue( - ImmutableList.of("value1", "value2", "value4", "value6", "value7") - .stream() - .allMatch(results::contains)); - - builder = DAGRequest.newBuilder(); - builder.setStartTs(1); - keyRanges = - ImmutableList.of( - createByteStringRange( - ByteString.copyFromUtf8("error1"), ByteString.copyFromUtf8("error2"))); - - server.putError("error1", KVMockServer.ABORT); - try { - coprocess(client, builder.build(), keyRanges); - fail(); - } catch (Exception e) { - assertTrue(true); - } - server.clearAllMap(); - client.close(); - } - - private SelectResponse coprocess( - RegionStoreClient client, DAGRequest request, List ranges) { - BackOffer backOffer = defaultBackOff(); - Queue responseQueue = new ArrayDeque<>(); - - client.coprocess(backOffer, request, ranges, responseQueue); - - List resultChunk = new ArrayList<>(); - while (!responseQueue.isEmpty()) { - SelectResponse response = responseQueue.poll(); - if (response != null) { - resultChunk.addAll(response.getChunksList()); - } - } - - return SelectResponse.newBuilder().addAllChunks(resultChunk).build(); - } - private BackOffer defaultBackOff() { return ConcreteBackOffer.newCustomBackOff(1000); } diff --git a/src/test/java/org/tikv/catalog/CatalogTest.java b/src/test/java/org/tikv/catalog/CatalogTest.java deleted file mode 100644 index c4a51532d3..0000000000 --- a/src/test/java/org/tikv/catalog/CatalogTest.java +++ /dev/null @@ -1,129 +0,0 @@ -package org.tikv.catalog; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.List; -import java.util.stream.Collectors; -import org.junit.Before; -import org.junit.Test; -import org.tikv.KVMockServer; -import org.tikv.PDMockServer; -import org.tikv.TiConfiguration; -import org.tikv.TiSession; -import org.tikv.kvproto.Kvrpcpb.CommandPri; -import org.tikv.kvproto.Kvrpcpb.IsolationLevel; -import org.tikv.meta.MetaUtils.MetaMockHelper; -import org.tikv.meta.TiDBInfo; -import org.tikv.meta.TiTableInfo; -import org.tikv.region.TiRegion; -import org.tikv.util.ReflectionWrapper; - -public class CatalogTest { - private KVMockServer kvServer; - private PDMockServer pdServer; - private static final long CLUSTER_ID = 1024; - private TiConfiguration conf; - - @Before - public void setUp() throws Exception { - pdServer = new PDMockServer(); - pdServer.start(CLUSTER_ID); - kvServer = new KVMockServer(); - kvServer.start( - new TiRegion( - MetaMockHelper.region, - MetaMockHelper.region.getPeers(0), - IsolationLevel.RC, - CommandPri.Low)); - // No PD needed in this test - conf = TiConfiguration.createDefault("127.0.0.1:" + pdServer.port); - } - - @Test - public void listDatabasesTest() throws Exception { - MetaMockHelper helper = new MetaMockHelper(pdServer, kvServer); - helper.preparePDForRegionRead(); - helper.setSchemaVersion(666); - - helper.addDatabase(130, "global_temp"); - helper.addDatabase(264, "TPCH_001"); - - TiSession session = TiSession.create(conf); - Catalog cat = session.getCatalog(); - List dbs = cat.listDatabases(); - List names = dbs.stream().map(TiDBInfo::getName).sorted().collect(Collectors.toList()); - assertEquals(2, dbs.size()); - assertEquals("global_temp", names.get(0)); - assertEquals("tpch_001", names.get(1)); - - helper.addDatabase(265, "other"); - helper.setSchemaVersion(667); - - ReflectionWrapper wrapper = new ReflectionWrapper(cat); - wrapper.call("reloadCache"); - - dbs = cat.listDatabases(); - assertEquals(3, dbs.size()); - names = dbs.stream().map(TiDBInfo::getName).sorted().collect(Collectors.toList()); - assertEquals("global_temp", names.get(0)); - assertEquals("other", names.get(1)); - assertEquals("tpch_001", names.get(2)); - - assertEquals(130, cat.getDatabase("global_temp").getId()); - assertEquals(null, cat.getDatabase("global_temp111")); - } - - @Test - public void listTablesTest() throws Exception { - MetaMockHelper helper = new MetaMockHelper(pdServer, kvServer); - helper.preparePDForRegionRead(); - helper.setSchemaVersion(666); - - helper.addDatabase(130, "global_temp"); - helper.addDatabase(264, "tpch_001"); - - helper.addTable(130, 42, "test"); - helper.addTable(130, 43, "test1"); - - TiSession session = TiSession.create(conf); - Catalog cat = session.getCatalog(); - TiDBInfo db = cat.getDatabase("gLObal_temp"); - List tables = cat.listTables(db); - List names = - tables.stream().map(TiTableInfo::getName).sorted().collect(Collectors.toList()); - assertEquals(2, tables.size()); - assertEquals("test", names.get(0)); - assertEquals("test1", names.get(1)); - - assertEquals("test", cat.getTable(db, 42).getName()); - assertEquals("test1", cat.getTable(db, 43).getName()); - assertEquals(null, cat.getTable(db, 44)); - - helper.addTable(130, 44, "other"); - helper.setSchemaVersion(667); - - ReflectionWrapper wrapper = new ReflectionWrapper(cat); - wrapper.call("reloadCache"); - - tables = cat.listTables(db); - names = tables.stream().map(TiTableInfo::getName).sorted().collect(Collectors.toList()); - assertEquals(3, tables.size()); - assertEquals("other", names.get(0)); - assertEquals("test", names.get(1)); - assertEquals("test1", names.get(2)); - - assertEquals(42, cat.getTable("global_temp", "test").getId()); - assertEquals(null, cat.getTable("global_temp", "test111")); - - helper.dropTable(db.getId(), tables.get(0).getId()); - helper.setSchemaVersion(668); - wrapper.call("reloadCache"); - tables = cat.listTables(db); - assertEquals(2, tables.size()); - - db = cat.getDatabase("TpCH_001"); - tables = cat.listTables(db); - assertTrue(tables.isEmpty()); - } -} diff --git a/src/test/java/org/tikv/catalog/CatalogTransactionTest.java b/src/test/java/org/tikv/catalog/CatalogTransactionTest.java deleted file mode 100644 index 75e1623fba..0000000000 --- a/src/test/java/org/tikv/catalog/CatalogTransactionTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.catalog; - -import static org.junit.Assert.assertEquals; - -import java.util.List; -import org.junit.Before; -import org.junit.Test; -import org.tikv.KVMockServer; -import org.tikv.PDMockServer; -import org.tikv.TiConfiguration; -import org.tikv.TiSession; -import org.tikv.kvproto.Kvrpcpb.CommandPri; -import org.tikv.kvproto.Kvrpcpb.IsolationLevel; -import org.tikv.meta.MetaUtils.MetaMockHelper; -import org.tikv.meta.TiDBInfo; -import org.tikv.meta.TiTableInfo; -import org.tikv.region.TiRegion; - -public class CatalogTransactionTest { - private KVMockServer kvServer; - private PDMockServer pdServer; - private static final long CLUSTER_ID = 1024; - private TiConfiguration conf; - - @Before - public void setUp() throws Exception { - pdServer = new PDMockServer(); - pdServer.start(CLUSTER_ID); - kvServer = new KVMockServer(); - kvServer.start( - new TiRegion( - MetaMockHelper.region, - MetaMockHelper.region.getPeers(0), - IsolationLevel.RC, - CommandPri.Low)); - // No PD needed in this test - conf = TiConfiguration.createDefault("127.0.0.1:" + pdServer.port); - } - - @Test - public void getLatestSchemaVersionTest() throws Exception { - MetaMockHelper helper = new MetaMockHelper(pdServer, kvServer); - helper.preparePDForRegionRead(); - helper.setSchemaVersion(666); - TiSession session = TiSession.create(conf); - CatalogTransaction trx = new CatalogTransaction(session.createSnapshot()); - assertEquals(666, trx.getLatestSchemaVersion()); - } - - @Test - public void getDatabasesTest() throws Exception { - MetaMockHelper helper = new MetaMockHelper(pdServer, kvServer); - helper.preparePDForRegionRead(); - helper.addDatabase(130, "global_temp"); - helper.addDatabase(264, "TPCH_001"); - - TiSession session = TiSession.create(conf); - CatalogTransaction trx = new CatalogTransaction(session.createSnapshot()); - List dbs = trx.getDatabases(); - assertEquals(2, dbs.size()); - assertEquals(130, dbs.get(0).getId()); - assertEquals("global_temp", dbs.get(0).getName()); - - assertEquals(264, dbs.get(1).getId()); - assertEquals("tpch_001", dbs.get(1).getName()); - - TiDBInfo db = trx.getDatabase(130); - assertEquals(130, db.getId()); - assertEquals("global_temp", db.getName()); - } - - @Test - public void getTablesTest() throws Exception { - MetaMockHelper helper = new MetaMockHelper(pdServer, kvServer); - helper.preparePDForRegionRead(); - helper.addTable(130, 42, "test"); - helper.addTable(130, 43, "test1"); - - TiSession session = TiSession.create(conf); - CatalogTransaction trx = new CatalogTransaction(session.createSnapshot()); - List tables = trx.getTables(130); - assertEquals(2, tables.size()); - assertEquals("test", tables.get(0).getName()); - assertEquals("test1", tables.get(1).getName()); - } -} diff --git a/src/test/java/org/tikv/codec/CodecTest.java b/src/test/java/org/tikv/codec/CodecTest.java index b7d84fb517..5a8cc242fc 100644 --- a/src/test/java/org/tikv/codec/CodecTest.java +++ b/src/test/java/org/tikv/codec/CodecTest.java @@ -15,78 +15,13 @@ package org.tikv.codec; -import static java.util.Objects.requireNonNull; import static org.junit.Assert.*; import static org.tikv.codec.Codec.*; -import com.google.common.primitives.UnsignedLong; -import java.math.BigDecimal; -import java.util.Calendar; -import java.util.TimeZone; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; import org.junit.Test; import org.tikv.codec.Codec.*; public class CodecTest { - @Test - public void writeDoubleAndReadDoubleTest() { - // issue scientific notation in toBin - CodecDataOutput cdo = new CodecDataOutput(); - RealCodec.writeDouble(cdo, 0.01); - double u = RealCodec.readDouble(new CodecDataInput(cdo.toBytes())); - assertEquals(0.01, u, 0.0001); - - cdo.reset(); - RealCodec.writeDouble(cdo, 206.0); - u = RealCodec.readDouble(new CodecDataInput(cdo.toBytes())); - assertEquals(206.0, u, 0.0001); - - cdo.reset(); - RealCodec.writeDouble(cdo, 0); - u = RealCodec.readDouble(new CodecDataInput(cdo.toBytes())); - assertEquals(0, u, 0.0001); - - cdo.reset(); - RealCodec.writeDouble(cdo, -0.01); - u = RealCodec.readDouble(new CodecDataInput(cdo.toBytes())); - assertEquals(-0.01, u, 0.0001); - - cdo.reset(); - RealCodec.writeDouble(cdo, -206.0); - u = RealCodec.readDouble(new CodecDataInput(cdo.toBytes())); - assertEquals(-206.0, u, 0.0001); - } - - @Test - public void readNWriteDecimalTest() throws Exception { - CodecDataOutput cdo = new CodecDataOutput(); - DecimalCodec.writeDecimal(cdo, BigDecimal.valueOf(0)); - BigDecimal bigDec = DecimalCodec.readDecimal(new CodecDataInput(cdo.toBytes())); - assertEquals(0, bigDec.doubleValue(), 0.0001); - - cdo.reset(); - DecimalCodec.writeDecimal(cdo, BigDecimal.valueOf(206.01)); - bigDec = DecimalCodec.readDecimal(new CodecDataInput(cdo.toBytes())); - assertEquals(206.01, bigDec.doubleValue(), 0.0001); - - cdo.reset(); - DecimalCodec.writeDecimal(cdo, BigDecimal.valueOf(-206.01)); - bigDec = DecimalCodec.readDecimal(new CodecDataInput(cdo.toBytes())); - assertEquals(-206.01, bigDec.doubleValue(), 0.0001); - - byte[] wrongData = new byte[] {(byte) 0x8}; - CodecDataInput cdi = new CodecDataInput(wrongData); - try { - DecimalCodec.readDecimal(cdi); - fail(); - } catch (Exception e) { - assertTrue(true); - } - } - @Test public void readNWriteLongTest() throws Exception { CodecDataOutput cdo = new CodecDataOutput(); @@ -279,251 +214,4 @@ public class CodecTest { // byte[] result = BytesCodec.readBytes(cdi, false); } - - @Test - public void writeFloatTest() throws Exception { - CodecDataOutput cdo = new CodecDataOutput(); - RealCodec.writeDouble(cdo, 0.00); - double u = RealCodec.readDouble(new CodecDataInput(cdo.toBytes())); - assertEquals(0.00, u, 0); - - cdo.reset(); - RealCodec.writeDouble(cdo, Double.MAX_VALUE); - u = RealCodec.readDouble(new CodecDataInput(cdo.toBytes())); - assertEquals(Double.MAX_VALUE, u, 0); - - cdo.reset(); - RealCodec.writeDouble(cdo, Double.MIN_VALUE); - u = RealCodec.readDouble(new CodecDataInput(cdo.toBytes())); - assertEquals(Double.MIN_VALUE, u, 0); - } - - @Test - public void readFloatTest() throws Exception { - byte[] data = - new byte[] { - (byte) (191 & 0xFF), - (byte) (241 & 0xFF), - (byte) (153 & 0xFF), - (byte) (153 & 0xFF), - (byte) (160 & 0xFF), - 0, - 0, - 0 - }; - CodecDataInput cdi = new CodecDataInput(data); - double u = RealCodec.readDouble(cdi); - assertEquals(1.1, u, 0.0001); - - data = - new byte[] { - (byte) (192 & 0xFF), - (byte) (1 & 0xFF), - (byte) (153 & 0xFF), - (byte) (153 & 0xFF), - (byte) (153 & 0xFF), - (byte) (153 & 0xFF), - (byte) (153 & 0xFF), - (byte) (154 & 0xFF) - }; - cdi = new CodecDataInput(data); - u = RealCodec.readDouble(cdi); - assertEquals(2.2, u, 0.0001); - - data = - new byte[] { - (byte) (63 & 0xFF), - (byte) (167 & 0xFF), - (byte) (51 & 0xFF), - (byte) (67 & 0xFF), - (byte) (159 & 0xFF), - (byte) (0xFF), - (byte) (0xFF), - (byte) (0xFF) - }; - - cdi = new CodecDataInput(data); - u = RealCodec.readDouble(cdi); - assertEquals(-99.199, u, 0.0001); - } - - @Test - public void negativeLongTest() throws Exception { - CodecDataOutput cdo = new CodecDataOutput(); - IntegerCodec.writeULong(cdo, UnsignedLong.valueOf("13831004815617530266").longValue()); - double u = RealCodec.readDouble(new CodecDataInput(cdo.toBytes())); - assertEquals(1.1, u, 0.001); - - cdo.reset(); - IntegerCodec.writeULong(cdo, UnsignedLong.valueOf("13835508415244900762").longValue()); - u = RealCodec.readDouble(new CodecDataInput(cdo.toBytes())); - assertEquals(2.2, u, 0.001); - - cdo.reset(); - IntegerCodec.writeULong(cdo, UnsignedLong.valueOf("13837985394932580352").longValue()); - u = RealCodec.readDouble(new CodecDataInput(cdo.toBytes())); - assertEquals(3.3, u, 0.001); - } - - @Test - public void fromPackedLongAndToPackedLongTest() { - DateTimeZone utc = DateTimeZone.UTC; - DateTimeZone otherTz = DateTimeZone.forOffsetHours(-8); - DateTime time = new DateTime(1999, 12, 12, 1, 1, 1, 999); - // Encode as UTC (loss timezone) and read it back as UTC - DateTime time1 = - requireNonNull(DateTimeCodec.fromPackedLong(DateTimeCodec.toPackedLong(time, utc), utc)); - assertEquals(time.getMillis(), time1.getMillis()); - - // Parse String as -8 timezone, encode and read it back - DateTimeFormatter formatter = - DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss:SSSSSSS").withZone(otherTz); - DateTime time2 = DateTime.parse("2010-10-10 10:11:11:0000000", formatter); - DateTime time3 = - requireNonNull( - DateTimeCodec.fromPackedLong(DateTimeCodec.toPackedLong(time2, otherTz), otherTz)); - assertEquals(time2.getMillis(), time3.getMillis()); - - // when packedLong is 0, then null is returned - DateTime time4 = DateTimeCodec.fromPackedLong(0, otherTz); - assertNull(time4); - - DateTime time5 = DateTime.parse("9999-12-31 23:59:59:0000000", formatter); - DateTime time6 = - requireNonNull( - DateTimeCodec.fromPackedLong(DateTimeCodec.toPackedLong(time5, otherTz), otherTz)); - assertEquals(time5.getMillis(), time6.getMillis()); - - DateTime time7 = DateTime.parse("1000-01-01 00:00:00:0000000", formatter); - DateTime time8 = - requireNonNull( - DateTimeCodec.fromPackedLong(DateTimeCodec.toPackedLong(time7, otherTz), otherTz)); - assertEquals(time7.getMillis(), time8.getMillis()); - - DateTime time9 = DateTime.parse("2017-01-05 23:59:59:5756010", formatter); - DateTime time10 = - requireNonNull( - DateTimeCodec.fromPackedLong(DateTimeCodec.toPackedLong(time9, otherTz), otherTz)); - assertEquals(time9.getMillis(), time10.getMillis()); - - DateTimeFormatter formatter1 = DateTimeFormat.forPattern("yyyy-MM-dd"); - DateTime date1 = DateTime.parse("2099-10-30", formatter1); - long time11 = DateTimeCodec.toPackedLong(date1, otherTz); - DateTime time12 = requireNonNull(DateTimeCodec.fromPackedLong(time11, otherTz)); - assertEquals(time12.getMillis(), date1.getMillis()); - - Calendar cal = Calendar.getInstance(otherTz.toTimeZone()); - cal.set(Calendar.HOUR_OF_DAY, 0); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - long millis = cal.getTimeInMillis(); - DateTime time14 = - requireNonNull( - DateTimeCodec.fromPackedLong(DateCodec.toPackedLong(millis, otherTz), otherTz)); - assertEquals(millis, time14.getMillis()); - } - - @Test - public void DSTTest() { - DateTimeZone defaultDateTimeZone = DateTimeZone.getDefault(); - DateTimeZone utc = DateTimeZone.UTC; - DateTimeZone dst = DateTimeZone.forID("America/New_York"); - TimeZone defaultTimeZone = TimeZone.getDefault(); - TimeZone utcTimeZone = TimeZone.getTimeZone("UTC"); - TimeZone dstTimeZone = TimeZone.getTimeZone("America/New_York"); - - TimeZone.setDefault(utcTimeZone); - DateTimeZone.setDefault(utc); - - DateTime time = new DateTime(2007, 3, 11, 2, 0, 0, 0); - DateTime time11 = new DateTime(2007, 3, 11, 7, 0, 0, 0).toDateTime(dst); - DateTime time21 = new DateTime(2007, 3, 11, 6, 0, 0, 0).toDateTime(dst); - - TimeZone.setDefault(dstTimeZone); - DateTimeZone.setDefault(dst); - - // Local Time 2007-03-11T02:00:00.000 should be in dst gap - assertTrue(dst.isLocalDateTimeGap(time.toLocalDateTime())); - - // Try Decode a dst datetime - DateTime time12 = - requireNonNull( - DateTimeCodec.fromPackedLong( - ((((2007L * 13 + 3) << 5L | 11) << 17L) | (2 << 12)) << 24L, dst)); - DateTime time22 = - requireNonNull( - DateTimeCodec.fromPackedLong( - ((((2007L * 13 + 3) << 5L | 11) << 17L) | (1 << 12)) << 24L, dst)); - DateTime time23 = - requireNonNull( - DateTimeCodec.fromPackedLong( - ((((2007L * 13 + 3) << 5L | 11) << 17L) | (7 << 12)) << 24L, utc)); - DateTime time24 = time23.toDateTime(dst); - // time11: 2007-03-11T03:00:00.000-04:00 - // time21: 2007-03-11T01:00:00.000-05:00 - // time12: 2007-03-11T03:00:00.000-04:00 - // time22: 2007-03-11T01:00:00.000-05:00 - // time23: 2007-03-11T07:00:00.000Z - // time24: 2007-03-11T03:00:00.000-04:00 - assertEquals(time11.getMillis(), time12.getMillis()); - assertEquals(time21.getMillis(), time22.getMillis()); - assertEquals(time12.getMillis(), time23.getMillis()); - assertEquals(time23.getMillis(), time24.getMillis()); - assertEquals(time22.getMillis() + 60 * 60 * 1000, time23.getMillis()); - - assertEquals(time12.toLocalDateTime().getHourOfDay(), 3); - assertEquals(time22.toLocalDateTime().getHourOfDay(), 1); - assertEquals(time23.toLocalDateTime().getHourOfDay(), 7); - assertEquals(time24.toLocalDateTime().getHourOfDay(), 3); - - TimeZone.setDefault(defaultTimeZone); - DateTimeZone.setDefault(defaultDateTimeZone); - } - - @Test - public void readNWriteDateTimeTest() { - DateTimeZone otherTz = DateTimeZone.forOffsetHours(-8); - DateTime time = new DateTime(2007, 3, 11, 2, 0, 0, 0, DateTimeZone.UTC); - - CodecDataOutput cdo = new CodecDataOutput(); - DateTimeCodec.writeDateTimeFully(cdo, time, otherTz); - DateTimeCodec.writeDateTimeProto(cdo, time, otherTz); - - assertArrayEquals( - new byte[] { - (byte) 0x4, - (byte) 0x19, - (byte) 0x7b, - (byte) 0x95, - (byte) 0x20, - (byte) 0x00, - (byte) 0x00, - (byte) 0x00, - (byte) 0x00, - (byte) 0x19, - (byte) 0x7b, - (byte) 0x95, - (byte) 0x20, - (byte) 0x00, - (byte) 0x00, - (byte) 0x00, - (byte) 0x00, - }, - cdo.toBytes()); - - CodecDataInput cdi = new CodecDataInput(cdo.toBytes()); - assertEquals(UINT_FLAG, cdi.readByte()); - DateTime time2 = DateTimeCodec.readFromUInt(cdi, otherTz); - assertEquals(time.getMillis(), time2.getMillis()); - time2 = DateTimeCodec.readFromUInt(cdi, otherTz); - assertEquals(time.getMillis(), time2.getMillis()); - - cdo.reset(); - IntegerCodec.writeULongFully(cdo, DateTimeCodec.toPackedLong(time, otherTz), false); - cdi = new CodecDataInput(cdo.toBytes()); - assertEquals(UVARINT_FLAG, cdi.readByte()); - time2 = DateTimeCodec.readFromUVarInt(cdi, otherTz); - assertEquals(time.getMillis(), time2.getMillis()); - } } diff --git a/src/test/java/org/tikv/expression/ComparisonBinaryExpressionTest.java b/src/test/java/org/tikv/expression/ComparisonBinaryExpressionTest.java deleted file mode 100644 index 7a8d60e30f..0000000000 --- a/src/test/java/org/tikv/expression/ComparisonBinaryExpressionTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression; - -import static org.junit.Assert.assertEquals; -import static org.tikv.expression.ArithmeticBinaryExpression.divide; -import static org.tikv.expression.ComparisonBinaryExpression.NormalizedPredicate; -import static org.tikv.expression.ComparisonBinaryExpression.Type; -import static org.tikv.expression.ComparisonBinaryExpression.equal; -import static org.tikv.expression.ComparisonBinaryExpression.greaterEqual; -import static org.tikv.expression.ComparisonBinaryExpression.greaterThan; -import static org.tikv.expression.ComparisonBinaryExpression.lessEqual; -import static org.tikv.expression.ComparisonBinaryExpression.lessThan; -import static org.tikv.expression.ComparisonBinaryExpression.notEqual; - -import com.google.common.collect.ImmutableList; -import org.junit.Test; -import org.tikv.key.TypedKey; -import org.tikv.meta.MetaUtils; -import org.tikv.meta.TiTableInfo; -import org.tikv.types.DataType; -import org.tikv.types.IntegerType; -import org.tikv.types.StringType; - -public class ComparisonBinaryExpressionTest { - private static TiTableInfo createTable() { - return new MetaUtils.TableBuilder() - .name("testTable") - .addColumn("c1", IntegerType.INT, true) - .addColumn("c2", StringType.VARCHAR) - .addColumn("c3", StringType.VARCHAR) - .addColumn("c4", IntegerType.INT) - .appendIndex("testIndex", ImmutableList.of("c1", "c2"), false) - .build(); - } - - private void verifyNormalize( - ComparisonBinaryExpression cond, String colName, Object value, DataType dataType, Type type) { - NormalizedPredicate normCond = cond.normalize(); - assertEquals(colName, normCond.getColumnRef().getName()); - assertEquals(value, normCond.getValue().getValue()); - assertEquals(TypedKey.toTypedKey(value, dataType), normCond.getTypedLiteral()); - assertEquals(type, normCond.getType()); - } - - @Test - public void normalizeTest() throws Exception { - TiTableInfo table = createTable(); - ColumnRef col1 = ColumnRef.create("c1", table); - Constant c1 = Constant.create(1, IntegerType.INT); - // index col = c1, long - ComparisonBinaryExpression cond = equal(col1, c1); - verifyNormalize(cond, "c1", 1, IntegerType.INT, Type.EQUAL); - - cond = lessEqual(c1, col1); - verifyNormalize(cond, "c1", 1, IntegerType.INT, Type.GREATER_EQUAL); - - cond = lessThan(c1, col1); - verifyNormalize(cond, "c1", 1, IntegerType.INT, Type.GREATER_THAN); - - cond = greaterEqual(c1, col1); - verifyNormalize(cond, "c1", 1, IntegerType.INT, Type.LESS_EQUAL); - - cond = greaterThan(c1, col1); - verifyNormalize(cond, "c1", 1, IntegerType.INT, Type.LESS_THAN); - - cond = equal(c1, col1); - verifyNormalize(cond, "c1", 1, IntegerType.INT, Type.EQUAL); - - cond = notEqual(c1, col1); - verifyNormalize(cond, "c1", 1, IntegerType.INT, Type.NOT_EQUAL); - - cond = lessEqual(col1, c1); - verifyNormalize(cond, "c1", 1, IntegerType.INT, Type.LESS_EQUAL); - - cond = equal(divide(col1, c1), c1); - assertEquals(null, cond.normalize()); - } -} diff --git a/src/test/java/org/tikv/expression/visitor/ExpressionTypeCoercerTest.java b/src/test/java/org/tikv/expression/visitor/ExpressionTypeCoercerTest.java deleted file mode 100644 index ac1d462a7d..0000000000 --- a/src/test/java/org/tikv/expression/visitor/ExpressionTypeCoercerTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.expression.visitor; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; -import static org.tikv.expression.ArithmeticBinaryExpression.minus; -import static org.tikv.expression.ArithmeticBinaryExpression.plus; -import static org.tikv.expression.ComparisonBinaryExpression.equal; -import static org.tikv.expression.ComparisonBinaryExpression.lessThan; -import static org.tikv.expression.LogicalBinaryExpression.and; - -import com.google.common.collect.ImmutableList; -import java.util.Map; -import org.junit.Test; -import org.tikv.expression.ArithmeticBinaryExpression; -import org.tikv.expression.ColumnRef; -import org.tikv.expression.ComparisonBinaryExpression; -import org.tikv.expression.Constant; -import org.tikv.expression.Expression; -import org.tikv.expression.LogicalBinaryExpression; -import org.tikv.meta.MetaUtils; -import org.tikv.meta.TiTableInfo; -import org.tikv.types.DataType; -import org.tikv.types.IntegerType; -import org.tikv.types.RealType; -import org.tikv.types.StringType; -import org.tikv.types.TimestampType; - -public class ExpressionTypeCoercerTest { - private static TiTableInfo createTable() { - return new MetaUtils.TableBuilder() - .name("testTable") - .addColumn("c1", IntegerType.INT, true) - .addColumn("c2", StringType.VARCHAR) - .addColumn("c3", TimestampType.TIMESTAMP) - .addColumn("c4", RealType.DOUBLE) - .appendIndex("testIndex", ImmutableList.of("c1", "c2"), false) - .build(); - } - - @Test - public void typeVerifyWithColumnRefTest() throws Exception { - TiTableInfo table = createTable(); - ColumnRef col1 = ColumnRef.create("c1", table); // INT - ColumnRef col4 = ColumnRef.create("c4", table); // DOUBLE - Constant c1 = Constant.create(1); - Constant c2 = Constant.create(11.1); - Constant c3 = Constant.create(11.1); - Constant c4 = Constant.create(1.1); - - ArithmeticBinaryExpression ar1 = minus(col1, c1); - ArithmeticBinaryExpression ar2 = plus(col4, c4); - ComparisonBinaryExpression comp1 = equal(ar1, c2); - ComparisonBinaryExpression comp2 = equal(ar2, c3); - ComparisonBinaryExpression comp3 = equal(ar1, ar2); - LogicalBinaryExpression log1 = and(comp1, comp2); - LogicalBinaryExpression log2 = and(comp1, comp3); - - ExpressionTypeCoercer inf = new ExpressionTypeCoercer(); - assertEquals(IntegerType.BOOLEAN, inf.infer(log1)); - Map map = inf.getTypeMap(); - assertEquals(IntegerType.INT, map.get(col1)); - assertEquals(IntegerType.INT, map.get(c1)); - assertEquals(IntegerType.INT, map.get(ar1)); - assertEquals(IntegerType.BOOLEAN, map.get(comp1)); - assertEquals(RealType.DOUBLE, map.get(col4)); - assertEquals(RealType.DOUBLE, map.get(c4)); - assertEquals(RealType.DOUBLE, map.get(ar2)); - assertEquals(IntegerType.BOOLEAN, map.get(comp2)); - - inf = new ExpressionTypeCoercer(); - try { - inf.infer(log2); - fail(); - } catch (Exception e) { - } - } - - @Test - public void typeVerifyTest() throws Exception { - Constant const1 = Constant.create(1); - Constant const2 = Constant.create(11); - ComparisonBinaryExpression comp1 = equal(const1, const2); - - Constant const3 = Constant.create(1.1f); - Constant const4 = Constant.create(1.111f); - ComparisonBinaryExpression comp2 = lessThan(const3, const4); - - Constant const5 = Constant.create(1); - Constant const6 = Constant.create(1.1f); - ArithmeticBinaryExpression comp3 = minus(const5, const6); - - LogicalBinaryExpression and1 = and(comp1, comp2); - LogicalBinaryExpression or1 = LogicalBinaryExpression.or(comp1, comp3); - - ExpressionTypeCoercer inf = new ExpressionTypeCoercer(); - assertEquals(IntegerType.BOOLEAN, inf.infer(and1)); - Map map = inf.getTypeMap(); - assertEquals(IntegerType.BIGINT, map.get(const1)); - assertEquals(IntegerType.BIGINT, map.get(const2)); - assertEquals(RealType.FLOAT, map.get(const3)); - assertEquals(RealType.FLOAT, map.get(const4)); - assertEquals(IntegerType.BOOLEAN, map.get(comp1)); - assertEquals(IntegerType.BOOLEAN, map.get(comp2)); - assertEquals(IntegerType.BOOLEAN, map.get(and1)); - assertEquals(IntegerType.BIGINT, inf.infer(comp3)); // for now, we unify type from left to right - assertEquals(IntegerType.BOOLEAN, inf.infer(or1)); - } -} diff --git a/src/test/java/org/tikv/key/CompoundKeyTest.java b/src/test/java/org/tikv/key/CompoundKeyTest.java deleted file mode 100644 index 44e8c0eba0..0000000000 --- a/src/test/java/org/tikv/key/CompoundKeyTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.key; - -import static org.junit.Assert.assertEquals; - -import org.junit.Test; - -public class CompoundKeyTest { - - @Test - public void concatTest() throws Exception { - Key k1 = Key.toRawKey(new byte[] {1, 2, 3}); - Key k2 = Key.toRawKey(new byte[] {4, 5, 6}); - CompoundKey cpk1 = CompoundKey.concat(k1, k2); - Key k3 = Key.toRawKey(new byte[] {4}); - Key k4 = Key.toRawKey(new byte[] {5, 6}); - - assertEquals(Key.toRawKey(new byte[] {1, 2, 3, 4, 5, 6}), cpk1); - - CompoundKey cpk2 = CompoundKey.concat(k1, k3); - CompoundKey cpk3 = CompoundKey.concat(cpk2, k4); - assertEquals(cpk1, cpk3); - } - - @Test - public void getKeysTest() throws Exception { - Key k1 = Key.toRawKey(new byte[] {1, 2, 3}); - Key k2 = Key.toRawKey(new byte[] {4, 5, 6}); - Key k3 = Key.toRawKey(new byte[] {7, 8, 9}); - CompoundKey.Builder b1 = CompoundKey.newBuilder(); - CompoundKey cpk1 = b1.append(k1).append(k2).build(); - assertEquals(2, cpk1.getKeys().size()); - assertEquals(k1, cpk1.getKeys().get(0)); - assertEquals(k2, cpk1.getKeys().get(1)); - assertEquals("[{1,2,3},{4,5,6}]", cpk1.toString()); - - CompoundKey.Builder b2 = CompoundKey.newBuilder(); - CompoundKey cpk2 = b2.append(cpk1).append(k3).build(); - assertEquals(3, cpk2.getKeys().size()); - assertEquals(k1, cpk2.getKeys().get(0)); - assertEquals(k2, cpk2.getKeys().get(1)); - assertEquals(k3, cpk2.getKeys().get(2)); - assertEquals("[{1,2,3},{4,5,6},{7,8,9}]", cpk2.toString()); - } -} diff --git a/src/test/java/org/tikv/key/IndexKeyTest.java b/src/test/java/org/tikv/key/IndexKeyTest.java deleted file mode 100644 index 3678735de6..0000000000 --- a/src/test/java/org/tikv/key/IndexKeyTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.key; - -import static org.junit.Assert.*; - -import org.junit.Test; -import org.tikv.types.IntegerType; - -public class IndexKeyTest { - - @Test - public void createTest() throws Exception { - Key k1 = Key.toRawKey(new byte[] {1, 2, 3, 4}); - Key k2 = Key.toRawKey(new byte[] {5, 6, 7, 8}); - Key k3 = Key.toRawKey(new byte[] {5, 6, 7, 9}); - IndexKey ik1 = IndexKey.toIndexKey(666, 777, k1, k2); - IndexKey ik2 = IndexKey.toIndexKey(666, 777, k1, k2); - IndexKey ik3 = IndexKey.toIndexKey(666, 777, k1, k3); - assertEquals(ik1, ik2); - assertTrue(ik1.compareTo(ik2) == 0); - assertTrue(ik1.compareTo(ik3) < 0); - assertEquals(2, ik1.getDataKeys().length); - assertEquals(k1, ik1.getDataKeys()[0]); - assertEquals(k2, ik1.getDataKeys()[1]); - - try { - IndexKey.toIndexKey(0, 0, k1, null, k2); - fail(); - } catch (Exception e) { - } - } - - @Test - public void toStringTest() throws Exception { - Key k1 = Key.toRawKey(new byte[] {1, 2, 3, 4}); - TypedKey k2 = TypedKey.toTypedKey(666, IntegerType.INT); - IndexKey ik = IndexKey.toIndexKey(0, 0, k1, Key.NULL, k2); - assertEquals("[{1,2,3,4},null,666]", ik.toString()); - } -} diff --git a/src/test/java/org/tikv/key/KeyTest.java b/src/test/java/org/tikv/key/KeyTest.java index e18fd1bf95..c391027294 100644 --- a/src/test/java/org/tikv/key/KeyTest.java +++ b/src/test/java/org/tikv/key/KeyTest.java @@ -19,50 +19,9 @@ import static org.junit.Assert.*; import static org.tikv.key.Key.toRawKey; import com.google.common.primitives.UnsignedBytes; -import com.google.protobuf.ByteString; -import java.util.Arrays; -import java.util.function.Function; import org.junit.Test; -import org.tikv.types.DataType; -import org.tikv.types.IntegerType; public class KeyTest { - @Test - public void toKeyTest() throws Exception { - // compared as unsigned - testBytes(new byte[] {1, 2, -1, 10}, new byte[] {1, 2, 0, 10}, x -> x > 0); - testBytes(new byte[] {1, 2, 0, 10}, new byte[] {1, 2, 0, 10}, x -> x == 0); - testBytes(new byte[] {1, 2, 0, 10}, new byte[] {1, 2, 1, 10}, x -> x < 0); - testBytes(new byte[] {1, 2, 0, 10}, new byte[] {1, 2, 0}, x -> x > 0); - - testLiteral(1, 2, IntegerType.INT, x -> x < 0); - testLiteral(13, 13, IntegerType.INT, x -> x == 0); - testLiteral(13, 2, IntegerType.INT, x -> x > 0); - testLiteral(-1, 2, IntegerType.INT, x -> x < 0); - } - - private void testBytes(byte[] lhs, byte[] rhs, Function tester) { - ByteString lhsBS = ByteString.copyFrom(lhs); - ByteString rhsBS = ByteString.copyFrom(rhs); - - Key lhsComp = toRawKey(lhsBS); - Key rhsComp = toRawKey(rhsBS); - - assertTrue(tester.apply(lhsComp.compareTo(rhsComp))); - - lhsComp = toRawKey(lhs); - rhsComp = toRawKey(rhs); - - assertTrue(tester.apply(lhsComp.compareTo(rhsComp))); - } - - private void testLiteral( - Object lhs, Object rhs, DataType type, Function tester) { - Key lhsComp = TypedKey.toTypedKey(lhs, type); - Key rhsComp = TypedKey.toTypedKey(rhs, type); - - assertTrue(tester.apply(lhsComp.compareTo(rhsComp))); - } @Test public void nextTest() throws Exception { @@ -73,16 +32,4 @@ public class KeyTest { assertEquals( toRawKey(new byte[] {UnsignedBytes.MAX_VALUE, UnsignedBytes.MAX_VALUE, 0}), k1.next()); } - - @Test - public void compareToTest() throws Exception { - Key kNegInf = toRawKey(new byte[0], true); - Key kMin = Key.MIN; - Key k = toRawKey(new byte[] {1}); - Key kMax = Key.MAX; - Key kInf = toRawKey(new byte[0], false); - Key[] keys = new Key[] {kInf, kMax, k, kMin, kNegInf}; - Arrays.sort(keys); - assertArrayEquals(new Key[] {kNegInf, kMin, k, kMax, kInf}, keys); - } } diff --git a/src/test/java/org/tikv/meta/DBInfoTest.java b/src/test/java/org/tikv/meta/DBInfoTest.java deleted file mode 100644 index 6a04e89e63..0000000000 --- a/src/test/java/org/tikv/meta/DBInfoTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import static org.junit.Assert.*; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.Test; - -public class DBInfoTest { - @Test - public void testSerialize() throws Exception { - ObjectMapper mapper = new ObjectMapper(); - String json = - "{\"id\":1,\"db_name\":{\"O\":\"test\",\"L\":\"test\"},\"charset\":\"utf8\",\"collate\":\"utf8_bin\",\"state\":5}"; - TiDBInfo dbInfo = mapper.readValue(json, TiDBInfo.class); - assertEquals(dbInfo.getId(), 1); - assertEquals(dbInfo.getName(), "test"); - assertEquals(dbInfo.getCharset(), "utf8"); - assertEquals(dbInfo.getCollate(), "utf8_bin"); - assertEquals(dbInfo.getSchemaState(), SchemaState.StatePublic); - } -} diff --git a/src/test/java/org/tikv/meta/MetaUtils.java b/src/test/java/org/tikv/meta/MetaUtils.java deleted file mode 100644 index ebae97a8c4..0000000000 --- a/src/test/java/org/tikv/meta/MetaUtils.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import com.google.common.collect.ImmutableList; -import com.google.protobuf.ByteString; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.tikv.GrpcUtils; -import org.tikv.KVMockServer; -import org.tikv.PDMockServer; -import org.tikv.codec.Codec.BytesCodec; -import org.tikv.codec.Codec.IntegerCodec; -import org.tikv.codec.CodecDataOutput; -import org.tikv.exception.TiClientInternalException; -import org.tikv.kvproto.Metapb; -import org.tikv.types.DataType; - -public class MetaUtils { - public static class TableBuilder { - static long autoId = 1; - - private static long newId() { - return autoId++; - } - - public static TableBuilder newBuilder() { - return new TableBuilder(); - } - - private boolean pkHandle; - private String name; - private List columns = new ArrayList<>(); - private List indices = new ArrayList<>(); - private Long tid = null; - - public TableBuilder() {} - - public TableBuilder name(String name) { - this.name = name; - return this; - } - - public TableBuilder tableId(long id) { - this.tid = id; - return this; - } - - public TableBuilder addColumn(String name, DataType type) { - return addColumn(name, type, false); - } - - public TableBuilder addColumn(String name, DataType type, boolean pk) { - for (TiColumnInfo c : columns) { - if (c.matchName(name)) { - throw new TiClientInternalException("duplicated name: " + name); - } - } - - TiColumnInfo col = new TiColumnInfo(newId(), name, columns.size(), type, pk); - columns.add(col); - return this; - } - - public TableBuilder appendIndex( - long iid, String indexName, List colNames, boolean isPk) { - List indexCols = - colNames - .stream() - .map(name -> columns.stream().filter(c -> c.matchName(name)).findFirst()) - .flatMap(col -> col.isPresent() ? Stream.of(col.get()) : Stream.empty()) - .map(TiColumnInfo::toIndexColumn) - .collect(Collectors.toList()); - - TiIndexInfo index = - new TiIndexInfo( - iid, - CIStr.newCIStr(indexName), - CIStr.newCIStr(name), - ImmutableList.copyOf(indexCols), - false, - isPk, - SchemaState.StatePublic.getStateCode(), - "", - IndexType.IndexTypeBtree.getTypeCode(), - false); - indices.add(index); - return this; - } - - public TableBuilder appendIndex(String indexName, List colNames, boolean isPk) { - return appendIndex(newId(), indexName, colNames, isPk); - } - - public TableBuilder setPkHandle(boolean pkHandle) { - this.pkHandle = pkHandle; - return this; - } - - public TiTableInfo build() { - if (tid == null) { - tid = newId(); - } - if (name == null) { - name = "Table" + tid; - } - return new TiTableInfo( - tid, CIStr.newCIStr(name), "", "", pkHandle, columns, indices, "", 0, 0, 0, 0, null); - } - } - - public static class MetaMockHelper { - public static final String LOCAL_ADDR = "127.0.0.1"; - public static int MEMBER_ID = 1; - public static int STORE_ID = 1; - public static Metapb.Region region = - Metapb.Region.newBuilder() - .setRegionEpoch(Metapb.RegionEpoch.newBuilder().setConfVer(1).setVersion(1)) - .setId(1) - .setStartKey(ByteString.EMPTY) - .setEndKey(ByteString.EMPTY) - .addPeers(Metapb.Peer.newBuilder().setId(1).setStoreId(1)) - .build(); - - public MetaMockHelper(PDMockServer pdServer, KVMockServer kvServer) { - this.kvServer = kvServer; - this.pdServer = pdServer; - } - - private KVMockServer kvServer; - private PDMockServer pdServer; - - public void preparePDForRegionRead() { - pdServer.addGetMemberResp( - GrpcUtils.makeGetMembersResponse( - pdServer.getClusterId(), - GrpcUtils.makeMember(MEMBER_ID, "http://" + LOCAL_ADDR + ":" + pdServer.port))); - - pdServer.addGetStoreResp( - GrpcUtils.makeGetStoreResponse( - pdServer.getClusterId(), - GrpcUtils.makeStore( - STORE_ID, LOCAL_ADDR + ":" + kvServer.getPort(), Metapb.StoreState.Up))); - - pdServer.addGetRegionResp(GrpcUtils.makeGetRegionResponse(pdServer.getClusterId(), region)); - } - - private ByteString getDBKey(long id) { - CodecDataOutput cdo = new CodecDataOutput(); - cdo.write(new byte[] {'m'}); - BytesCodec.writeBytes(cdo, "DBs".getBytes()); - IntegerCodec.writeULong(cdo, 'h'); - BytesCodec.writeBytes(cdo, String.format("DB:%d", id).getBytes()); - return cdo.toByteString(); - } - - public void addDatabase(long id, String name) { - String dbJson = - String.format( - "{\n" - + " \"id\":%d,\n" - + " \"db_name\":{\"O\":\"%s\",\"L\":\"%s\"},\n" - + " \"charset\":\"utf8\",\"collate\":\"utf8_bin\",\"state\":5\n" - + "}", - id, name, name.toLowerCase()); - - kvServer.put(getDBKey(id), ByteString.copyFromUtf8(dbJson)); - } - - public void dropDatabase(long id) { - kvServer.remove(getDBKey(id)); - } - - private ByteString getKeyForTable(long dbId, long tableId) { - ByteString dbKey = ByteString.copyFrom(String.format("%s:%d", "DB", dbId).getBytes()); - ByteString tableKey = - ByteString.copyFrom(String.format("%s:%d", "Table", tableId).getBytes()); - - CodecDataOutput cdo = new CodecDataOutput(); - cdo.write(new byte[] {'m'}); - BytesCodec.writeBytes(cdo, dbKey.toByteArray()); - IntegerCodec.writeULong(cdo, 'h'); - BytesCodec.writeBytes(cdo, tableKey.toByteArray()); - return cdo.toByteString(); - } - - private ByteString getSchemaVersionKey() { - CodecDataOutput cdo = new CodecDataOutput(); - cdo.write(new byte[] {'m'}); - BytesCodec.writeBytes(cdo, "SchemaVersionKey".getBytes()); - IntegerCodec.writeULong(cdo, 's'); - return cdo.toByteString(); - } - - public void setSchemaVersion(long version) { - CodecDataOutput cdo = new CodecDataOutput(); - cdo.write(new byte[] {'m'}); - BytesCodec.writeBytes(cdo, "SchemaVersionKey".getBytes()); - IntegerCodec.writeULong(cdo, 's'); - kvServer.put(getSchemaVersionKey(), ByteString.copyFromUtf8(String.format("%d", version))); - } - - public void addTable(int dbId, int tableId, String tableName) { - String tableJson = - String.format( - "\n" - + "{\n" - + " \"id\": %d,\n" - + " \"name\": {\n" - + " \"O\": \"%s\",\n" - + " \"L\": \"%s\"\n" - + " },\n" - + " \"charset\": \"\",\n" - + " \"collate\": \"\",\n" - + " \"cols\": [\n" - + " {\n" - + " \"id\": 1,\n" - + " \"name\": {\n" - + " \"O\": \"c1\",\n" - + " \"L\": \"c1\"\n" - + " },\n" - + " \"offset\": 0,\n" - + " \"origin_default\": null,\n" - + " \"default\": null,\n" - + " \"type\": {\n" - + " \"Tp\": 3,\n" - + " \"Flag\": 139,\n" - + " \"Flen\": 11,\n" - + " \"Decimal\": -1,\n" - + " \"Charset\": \"binary\",\n" - + " \"Collate\": \"binary\",\n" - + " \"Elems\": null\n" - + " },\n" - + " \"state\": 5,\n" - + " \"comment\": \"\"\n" - + " }\n" - + " ],\n" - + " \"index_info\": [],\n" - + " \"fk_info\": null,\n" - + " \"state\": 5,\n" - + " \"pk_is_handle\": true,\n" - + " \"comment\": \"\",\n" - + " \"auto_inc_id\": 0,\n" - + " \"max_col_id\": 4,\n" - + " \"max_idx_id\": 1\n" - + "}", - tableId, tableName, tableName.toLowerCase()); - - kvServer.put(getKeyForTable(dbId, tableId), ByteString.copyFromUtf8(tableJson)); - } - - public void dropTable(long dbId, long tableId) { - kvServer.remove(getKeyForTable(dbId, tableId)); - } - } -} diff --git a/src/test/java/org/tikv/meta/TiDAGRequestTest.java b/src/test/java/org/tikv/meta/TiDAGRequestTest.java deleted file mode 100644 index 4774924ac5..0000000000 --- a/src/test/java/org/tikv/meta/TiDAGRequestTest.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.tikv.expression.ArithmeticBinaryExpression.plus; -import static org.tikv.expression.ComparisonBinaryExpression.lessEqual; - -import com.google.common.collect.ImmutableList; -import com.google.protobuf.ByteString; -import com.pingcap.tidb.tipb.Expr; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.util.HashMap; -import java.util.Map; -import org.junit.Test; -import org.tikv.expression.*; -import org.tikv.expression.AggregateFunction.FunctionType; -import org.tikv.expression.visitor.ExpressionTypeCoercer; -import org.tikv.expression.visitor.ProtoConverter; -import org.tikv.kvproto.Coprocessor; -import org.tikv.types.IntegerType; -import org.tikv.types.StringType; - -public class TiDAGRequestTest { - private static TiTableInfo createTable() { - return new MetaUtils.TableBuilder() - .name("testTable") - .addColumn("c1", IntegerType.INT, true) - .addColumn("c2", StringType.VARCHAR) - .addColumn("c3", StringType.VARCHAR) - .addColumn("c4", IntegerType.INT) - .appendIndex("testIndex", ImmutableList.of("c1", "c2"), false) - .build(); - } - - @Test - public void testTopNCouldPushDownLimit0() { - TiTableInfo table = createTable(); - TiDAGRequest dagRequest = new TiDAGRequest(TiDAGRequest.PushDownType.NORMAL); - ColumnRef col1 = ColumnRef.create("c1", table); - dagRequest.addOrderByItem(ByItem.create(col1, false)); - dagRequest.addRequiredColumn(col1); - dagRequest.setLimit(0); - dagRequest.setTableInfo(table); - dagRequest.setStartTs(1); - dagRequest.buildScan(false); - } - - @Test - public void testSerializable() throws Exception { - TiTableInfo table = createTable(); - TiDAGRequest selReq = new TiDAGRequest(TiDAGRequest.PushDownType.NORMAL); - Constant c1 = Constant.create(1L, IntegerType.BIGINT); - Constant c2 = Constant.create(2L, IntegerType.BIGINT); - ColumnRef col1 = ColumnRef.create("c1", table); - ColumnRef col2 = ColumnRef.create("c2", table); - ColumnRef col3 = ColumnRef.create("c3", table); - - AggregateFunction sum = AggregateFunction.newCall(FunctionType.Sum, col1); - AggregateFunction min = AggregateFunction.newCall(FunctionType.Min, col1); - - selReq - .addRequiredColumn(col1) - .addRequiredColumn(col2) - .addRequiredColumn(col3) - .addAggregate(sum, ExpressionTypeCoercer.inferType(sum)) - .addAggregate(min, ExpressionTypeCoercer.inferType(min)) - .addFilter(plus(c1, c2)) - .addGroupByItem(ByItem.create(ColumnRef.create("c2", table), true)) - .addOrderByItem(ByItem.create(ColumnRef.create("c3", table), false)) - .setTableInfo(table) - .setStartTs(666) - .setTruncateMode(TiDAGRequest.TruncateMode.IgnoreTruncation) - .setDistinct(true) - .setIndexInfo(table.getIndices().get(0)) - .setHaving(lessEqual(col3, c2)) - .setLimit(100) - .addRanges( - ImmutableList.of( - Coprocessor.KeyRange.newBuilder() - .setStart(ByteString.copyFromUtf8("startkey")) - .setEnd(ByteString.copyFromUtf8("endkey")) - .build())); - - ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream(byteOutStream); - oos.writeObject(selReq); - - ByteArrayInputStream byteInStream = new ByteArrayInputStream(byteOutStream.toByteArray()); - ObjectInputStream ois = new ObjectInputStream(byteInStream); - TiDAGRequest derselReq = (TiDAGRequest) ois.readObject(); - assertTrue(selectRequestEquals(selReq, derselReq)); - } - - private static boolean selectRequestEquals(TiDAGRequest lhs, TiDAGRequest rhs) { - assertEquals(lhs.getFields().size(), rhs.getFields().size()); - Map lhsMap = new HashMap<>(); - Map rhsMap = new HashMap<>(); - for (int i = 0; i < lhs.getFields().size(); i++) { - ColumnRef lCol = lhs.getFields().get(i); - ColumnRef rCol = rhs.getFields().get(i); - lhsMap.put(lCol, i); - rhsMap.put(rCol, i); - } - for (int i = 0; i < lhs.getFields().size(); i++) { - Expression lhsExpr = lhs.getFields().get(i); - Expression rhsExpr = rhs.getFields().get(i); - Expr lhsExprProto = ProtoConverter.toProto(lhsExpr, lhsMap); - Expr rhsExprProto = ProtoConverter.toProto(rhsExpr, rhsMap); - - if (!lhsExprProto.equals(rhsExprProto)) return false; - } - - assertEquals(lhs.getAggregates().size(), rhs.getAggregates().size()); - for (int i = 0; i < lhs.getAggregates().size(); i++) { - Expression lhsExpr = lhs.getAggregates().get(i); - Expression rhsExpr = rhs.getAggregates().get(i); - - Expr lhsExprProto = ProtoConverter.toProto(lhsExpr, lhsMap); - Expr rhsExprProto = ProtoConverter.toProto(rhsExpr, rhsMap); - - if (!lhsExprProto.equals(rhsExprProto)) return false; - } - - assertEquals(lhs.getGroupByItems().size(), rhs.getGroupByItems().size()); - for (int i = 0; i < lhs.getGroupByItems().size(); i++) { - ByItem lhsItem = lhs.getGroupByItems().get(i); - ByItem rhsItem = rhs.getGroupByItems().get(i); - if (!lhsItem.toProto(lhsMap).equals(rhsItem.toProto(rhsMap))) return false; - } - - assertEquals(lhs.getOrderByItems().size(), rhs.getOrderByItems().size()); - for (int i = 0; i < lhs.getOrderByItems().size(); i++) { - ByItem lhsItem = lhs.getOrderByItems().get(i); - ByItem rhsItem = rhs.getOrderByItems().get(i); - if (!lhsItem.toProto(lhsMap).equals(rhsItem.toProto(rhsMap))) return false; - } - - assertEquals(lhs.getRanges().size(), rhs.getRanges().size()); - for (int i = 0; i < lhs.getRanges().size(); i++) { - Coprocessor.KeyRange lhsItem = lhs.getRanges().get(i); - Coprocessor.KeyRange rhsItem = rhs.getRanges().get(i); - if (!lhsItem.equals(rhsItem)) return false; - } - - assertEquals(lhs.getFilters().size(), rhs.getFilters().size()); - for (int i = 0; i < lhs.getFilters().size(); i++) { - Expression lhsItem = lhs.getFilters().get(i); - Expression rhsItem = rhs.getFilters().get(i); - - Expr lhsExprProto = ProtoConverter.toProto(lhsItem); - Expr rhsExprProto = ProtoConverter.toProto(rhsItem); - - if (!lhsExprProto.equals(rhsExprProto)) return false; - } - - assertEquals(lhs.getTableInfo().toProto(), rhs.getTableInfo().toProto()); - assertEquals(lhs.getLimit(), rhs.getLimit()); - assertEquals(lhs.isDistinct(), rhs.isDistinct()); - assertEquals( - lhs.getIndexInfo().toProto(lhs.getTableInfo()), - rhs.getIndexInfo().toProto(rhs.getTableInfo())); - assertEquals(lhs.getStartTs(), rhs.getStartTs()); - assertEquals(lhs.getTimeZoneOffset(), rhs.getTimeZoneOffset()); - assertEquals(lhs.getFlags(), rhs.getFlags()); - return true; - } -} diff --git a/src/test/java/org/tikv/meta/TiTableInfoTest.java b/src/test/java/org/tikv/meta/TiTableInfoTest.java deleted file mode 100644 index ab09f5ebf7..0000000000 --- a/src/test/java/org/tikv/meta/TiTableInfoTest.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.meta; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.*; -import org.junit.Test; -import org.tikv.types.IntegerType; -import org.tikv.types.StringType; - -public class TiTableInfoTest { - public static final String tableJson = - "\n" - + "{\n" - + " \"id\": 42,\n" - + " \"name\": {\n" - + " \"O\": \"test\",\n" - + " \"L\": \"test\"\n" - + " },\n" - + " \"charset\": \"\",\n" - + " \"collate\": \"\",\n" - + " \"cols\": [\n" - + " {\n" - + " \"id\": 1,\n" - + " \"name\": {\n" - + " \"O\": \"c1\",\n" - + " \"L\": \"c1\"\n" - + " },\n" - + " \"offset\": 0,\n" - + " \"origin_default\": null,\n" - + " \"default\": null,\n" - + " \"type\": {\n" - + " \"Tp\": 3,\n" - + " \"Flag\": 139,\n" - + " \"Flen\": 11,\n" - + " \"Decimal\": -1,\n" - + " \"Charset\": \"binary\",\n" - + " \"Collate\": \"binary\",\n" - + " \"Elems\": null\n" - + " },\n" - + " \"state\": 5,\n" - + " \"comment\": \"\"\n" - + " },\n" - + " {\n" - + " \"id\": 2,\n" - + " \"name\": {\n" - + " \"O\": \"c2\",\n" - + " \"L\": \"c2\"\n" - + " },\n" - + " \"offset\": 1,\n" - + " \"origin_default\": null,\n" - + " \"default\": null,\n" - + " \"type\": {\n" - + " \"Tp\": 15,\n" - + " \"Flag\": 0,\n" - + " \"Flen\": 100,\n" - + " \"Decimal\": -1,\n" - + " \"Charset\": \"utf8\",\n" - + " \"Collate\": \"utf8_bin\",\n" - + " \"Elems\": null\n" - + " },\n" - + " \"state\": 5,\n" - + " \"comment\": \"\"\n" - + " },\n" - + " {\n" - + " \"id\": 3,\n" - + " \"name\": {\n" - + " \"O\": \"c3\",\n" - + " \"L\": \"c3\"\n" - + " },\n" - + " \"offset\": 2,\n" - + " \"origin_default\": null,\n" - + " \"default\": null,\n" - + " \"type\": {\n" - + " \"Tp\": 15,\n" - + " \"Flag\": 0,\n" - + " \"Flen\": 100,\n" - + " \"Decimal\": -1,\n" - + " \"Charset\": \"utf8\",\n" - + " \"Collate\": \"utf8_bin\",\n" - + " \"Elems\": null\n" - + " },\n" - + " \"state\": 5,\n" - + " \"comment\": \"\"\n" - + " },\n" - + " {\n" - + " \"id\": 4,\n" - + " \"name\": {\n" - + " \"O\": \"c4\",\n" - + " \"L\": \"c4\"\n" - + " },\n" - + " \"offset\": 3,\n" - + " \"origin_default\": null,\n" - + " \"default\": null,\n" - + " \"type\": {\n" - + " \"Tp\": 3,\n" - + " \"Flag\": 128,\n" - + " \"Flen\": 11,\n" - + " \"Decimal\": -1,\n" - + " \"Charset\": \"binary\",\n" - + " \"Collate\": \"binary\",\n" - + " \"Elems\": null\n" - + " },\n" - + " \"state\": 5,\n" - + " \"comment\": \"\"\n" - + " }\n" - + " ],\n" - + " \"index_info\": [\n" - + " {\n" - + " \"id\": 1,\n" - + " \"idx_name\": {\n" - + " \"O\": \"test_index\",\n" - + " \"L\": \"test_index\"\n" - + " },\n" - + " \"tbl_name\": {\n" - + " \"O\": \"\",\n" - + " \"L\": \"\"\n" - + " },\n" - + " \"idx_cols\": [\n" - + " {\n" - + " \"name\": {\n" - + " \"O\": \"c1\",\n" - + " \"L\": \"c1\"\n" - + " },\n" - + " \"offset\": 0,\n" - + " \"length\": -1\n" - + " },\n" - + " {\n" - + " \"name\": {\n" - + " \"O\": \"c2\",\n" - + " \"L\": \"c2\"\n" - + " },\n" - + " \"offset\": 1,\n" - + " \"length\": -1\n" - + " },\n" - + " {\n" - + " \"name\": {\n" - + " \"O\": \"c3\",\n" - + " \"L\": \"c3\"\n" - + " },\n" - + " \"offset\": 2,\n" - + " \"length\": -1\n" - + " }\n" - + " ],\n" - + " \"is_unique\": false,\n" - + " \"is_primary\": false,\n" - + " \"state\": 5,\n" - + " \"comment\": \"\",\n" - + " \"index_type\": 0\n" - + " }\n" - + " ],\n" - + " \"fk_info\": null,\n" - + " \"state\": 5,\n" - + " \"pk_is_handle\": true,\n" - + " \"comment\": \"\",\n" - + " \"auto_inc_id\": 0,\n" - + " \"max_col_id\": 4,\n" - + " \"max_idx_id\": 1\n" - + "}"; - - @Test - public void testFromJson() throws Exception { - ObjectMapper mapper = new ObjectMapper(); - TiTableInfo tableInfo = mapper.readValue(tableJson, TiTableInfo.class); - assertEquals("test", tableInfo.getName()); - assertEquals(4, tableInfo.getColumns().size()); - assertEquals("c1", tableInfo.getColumns().get(0).getName()); - assertEquals(IntegerType.class, tableInfo.getColumns().get(0).getType().getClass()); - assertEquals("c2", tableInfo.getColumns().get(1).getName()); - assertEquals(StringType.class, tableInfo.getColumns().get(1).getType().getClass()); - assertTrue(tableInfo.isPkHandle()); - } - - @Test - public void testPartitionInfo() throws Exception {} - - @Test - public void testPartitionDef() throws Exception {} - - @Test - public void testSerializable() throws Exception { - ObjectMapper mapper = new ObjectMapper(); - TiTableInfo tableInfo = mapper.readValue(tableJson, TiTableInfo.class); - ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream(byteOutStream); - oos.writeObject(tableInfo); - - ByteArrayInputStream byteInStream = new ByteArrayInputStream(byteOutStream.toByteArray()); - ObjectInputStream ois = new ObjectInputStream(byteInStream); - TiTableInfo deserTable = (TiTableInfo) ois.readObject(); - assertEquals(deserTable.toProto(), tableInfo.toProto()); - } -} diff --git a/src/test/java/org/tikv/operation/ChunkIteratorTest.java b/src/test/java/org/tikv/operation/ChunkIteratorTest.java deleted file mode 100644 index c607fab36d..0000000000 --- a/src/test/java/org/tikv/operation/ChunkIteratorTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.operation; - -import static org.junit.Assert.assertEquals; - -import com.google.protobuf.ByteString; -import com.pingcap.tidb.tipb.Chunk; -import com.pingcap.tidb.tipb.RowMeta; -import java.util.ArrayList; -import java.util.List; -import org.junit.Before; -import org.junit.Test; -import org.tikv.codec.CodecDataInput; -import org.tikv.operation.iterator.ChunkIterator; -import org.tikv.row.ObjectRowImpl; -import org.tikv.row.Row; -import org.tikv.types.DataType; -import org.tikv.types.IntegerType; -import org.tikv.types.StringType; - -public class ChunkIteratorTest { - private List chunks = new ArrayList<>(); - - @Before - public void setup() { - // 8 2 2 2 a 8 4 2 2 b 8 6 2 2 c - // 1 a 2 b 3 c - String chunkStr = "\b\u0002\u0002\u0002a\b\u0004\u0002\u0002b\b\u0006\u0002\u0002c"; - Chunk chunk = - Chunk.newBuilder() - .setRowsData(ByteString.copyFromUtf8(chunkStr)) - .addRowsMeta(0, RowMeta.newBuilder().setHandle(1).setLength(5)) - .addRowsMeta(1, RowMeta.newBuilder().setHandle(2).setLength(5)) - .addRowsMeta(2, RowMeta.newBuilder().setHandle(3).setLength(5)) - .build(); - chunks.add(chunk); - } - - private static void setValueToRow(CodecDataInput cdi, DataType type, int pos, Row row) { - if (type.isNextNull(cdi)) { - cdi.readUnsignedByte(); - row.setNull(pos); - } else { - row.set(pos, type, type.decode(cdi)); - } - } - - @Test - public void chunkTest() { - ChunkIterator chunkIterator = ChunkIterator.getRawBytesChunkIterator(chunks); - DataType bytes = StringType.VARCHAR; - DataType ints = IntegerType.INT; - Row row = ObjectRowImpl.create(6); - CodecDataInput cdi = new CodecDataInput(chunkIterator.next()); - setValueToRow(cdi, ints, 0, row); - setValueToRow(cdi, bytes, 1, row); - cdi = new CodecDataInput(chunkIterator.next()); - setValueToRow(cdi, ints, 2, row); - setValueToRow(cdi, bytes, 3, row); - cdi = new CodecDataInput(chunkIterator.next()); - setValueToRow(cdi, ints, 4, row); - setValueToRow(cdi, bytes, 5, row); - assertEquals(row.getLong(0), 1); - assertEquals(row.getString(1), "a"); - assertEquals(row.getLong(2), 2); - assertEquals(row.getString(3), "b"); - assertEquals(row.getLong(4), 3); - assertEquals(row.getString(5), "c"); - } -} diff --git a/src/test/java/org/tikv/operation/SchemaInferTest.java b/src/test/java/org/tikv/operation/SchemaInferTest.java deleted file mode 100644 index 9d40f95f65..0000000000 --- a/src/test/java/org/tikv/operation/SchemaInferTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.operation; - -import static org.junit.Assert.assertEquals; -import static org.tikv.expression.ArithmeticBinaryExpression.plus; -import static org.tikv.expression.visitor.ExpressionTypeCoercer.inferType; - -import com.google.protobuf.ByteString; -import java.util.List; -import org.junit.Test; -import org.tikv.catalog.CatalogTransaction; -import org.tikv.expression.AggregateFunction; -import org.tikv.expression.AggregateFunction.FunctionType; -import org.tikv.expression.ByItem; -import org.tikv.expression.ColumnRef; -import org.tikv.expression.Constant; -import org.tikv.expression.Expression; -import org.tikv.meta.TiDAGRequest; -import org.tikv.meta.TiTableInfo; -import org.tikv.types.DataType; -import org.tikv.types.DecimalType; -import org.tikv.types.StringType; - -public class SchemaInferTest { - private final String table29 = - "{\"id\":29,\"name\":{\"O\":\"t1\",\"L\":\"t1\"},\"charset\":\"\",\"collate\":\"\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"time\",\"L\":\"time\"},\"offset\":0,\"origin_default\":null,\"default\":null,\"type\":{\"Tp\":10,\"Flag\":128,\"Flen\":-1,\"Decimal\":-1,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\"},{\"id\":2,\"name\":{\"O\":\"number\",\"L\":\"number\"},\"offset\":1,\"origin_default\":null,\"default\":null,\"type\":{\"Tp\":3,\"Flag\":128,\"Flen\":-1,\"Decimal\":-1,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null},\"state\":5,\"comment\":\"\"},{\"id\":3,\"name\":{\"O\":\"name\",\"L\":\"name\"},\"offset\":2,\"origin_default\":null,\"default\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":-1,\"Decimal\":-1,\"Charset\":\"utf8\",\"Collate\":\"utf8_bin\",\"Elems\":null},\"state\":5,\"comment\":\"\"}],\"index_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handle\":false,\"comment\":\"\",\"auto_inc_id\":0,\"max_col_id\":3,\"max_idx_id\":0}"; - private final ByteString table29Bs = ByteString.copyFromUtf8(table29); - - private TiTableInfo table = CatalogTransaction.parseFromJson(table29Bs, TiTableInfo.class); - private Expression number = ColumnRef.create("number", table); - private ColumnRef name = ColumnRef.create("name", table); - private Expression sum = AggregateFunction.newCall(FunctionType.Sum, number); - private ByItem simpleGroupBy = ByItem.create(name, false); - private ByItem complexGroupBy = - ByItem.create(plus(name, Constant.create("1", StringType.VARCHAR)), false); - - @Test - public void simpleSelectSchemaInferTest() throws Exception { - // select name from t1; - TiDAGRequest tiDAGRequest = new TiDAGRequest(TiDAGRequest.PushDownType.NORMAL); - tiDAGRequest.getFields().add(name); - tiDAGRequest.setTableInfo(table); - tiDAGRequest.resolve(); - List dataTypes = SchemaInfer.create(tiDAGRequest).getTypes(); - assertEquals(1, dataTypes.size()); - assertEquals(StringType.VARCHAR.getClass(), dataTypes.get(0).getClass()); - } - - @Test - public void selectAggSchemaInferTest() throws Exception { - // select sum(number) from t1; - TiDAGRequest tiDAGRequest = new TiDAGRequest(TiDAGRequest.PushDownType.NORMAL); - tiDAGRequest.addAggregate(sum, inferType(sum)); - tiDAGRequest.setTableInfo(table); - tiDAGRequest.resolve(); - List dataTypes = SchemaInfer.create(tiDAGRequest).getTypes(); - assertEquals(1, dataTypes.size()); - assertEquals(DecimalType.DECIMAL.getClass(), dataTypes.get(0).getClass()); - } - - @Test - public void selectAggWithGroupBySchemaInferTest() throws Exception { - // select sum(number) from t1 group by name; - TiDAGRequest dagRequest = new TiDAGRequest(TiDAGRequest.PushDownType.NORMAL); - dagRequest.setTableInfo(table); - dagRequest.getFields().add(name); - dagRequest.addAggregate(sum, inferType(sum)); - dagRequest.getGroupByItems().add(simpleGroupBy); - dagRequest.resolve(); - List dataTypes = SchemaInfer.create(dagRequest).getTypes(); - assertEquals(2, dataTypes.size()); - assertEquals(DecimalType.DECIMAL.getClass(), dataTypes.get(0).getClass()); - assertEquals(StringType.VARCHAR.getClass(), dataTypes.get(1).getClass()); - } - - @Test - public void complexGroupBySelectTest() throws Exception { - // select sum(number) from t1 group by name + "1"; - TiDAGRequest dagRequest = new TiDAGRequest(TiDAGRequest.PushDownType.NORMAL); - dagRequest.setTableInfo(table); - dagRequest.getFields().add(name); - dagRequest.addAggregate(sum, inferType(sum)); - dagRequest.getGroupByItems().add(complexGroupBy); - dagRequest.resolve(); - List dataTypes = SchemaInfer.create(dagRequest).getTypes(); - assertEquals(2, dataTypes.size()); - assertEquals(DecimalType.DECIMAL.getClass(), dataTypes.get(0).getClass()); - assertEquals(StringType.VARCHAR.getClass(), dataTypes.get(1).getClass()); - } -} diff --git a/src/test/java/org/tikv/operation/iterator/DAGIteratorTest.java b/src/test/java/org/tikv/operation/iterator/DAGIteratorTest.java deleted file mode 100644 index 6c3ec88644..0000000000 --- a/src/test/java/org/tikv/operation/iterator/DAGIteratorTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.operation.iterator; - -import static junit.framework.TestCase.assertEquals; - -import com.google.common.collect.ImmutableList; -import com.google.protobuf.ByteString; -import java.util.List; -import org.junit.Test; -import org.tikv.GrpcUtils; -import org.tikv.KVMockServer; -import org.tikv.MockServerTest; -import org.tikv.codec.Codec.BytesCodec; -import org.tikv.codec.Codec.IntegerCodec; -import org.tikv.codec.CodecDataOutput; -import org.tikv.expression.ColumnRef; -import org.tikv.kvproto.Coprocessor.KeyRange; -import org.tikv.kvproto.Metapb; -import org.tikv.meta.MetaUtils; -import org.tikv.meta.TiDAGRequest; -import org.tikv.meta.TiDAGRequest.PushDownType; -import org.tikv.meta.TiTableInfo; -import org.tikv.operation.SchemaInfer; -import org.tikv.row.Row; -import org.tikv.types.IntegerType; -import org.tikv.types.StringType; -import org.tikv.util.RangeSplitter.RegionTask; - -public class DAGIteratorTest extends MockServerTest { - - private static TiTableInfo createTable() { - return new MetaUtils.TableBuilder() - .name("testTable") - .addColumn("c1", IntegerType.INT, true) - .addColumn("c2", StringType.VARCHAR) - .build(); - } - - @Test - public void staleEpochTest() throws Exception { - Metapb.Store store = - Metapb.Store.newBuilder() - .setAddress(LOCAL_ADDR + ":" + port) - .setId(1) - .setState(Metapb.StoreState.Up) - .build(); - - TiTableInfo table = createTable(); - TiDAGRequest req = new TiDAGRequest(PushDownType.NORMAL); - req.setTableInfo(table); - req.addRequiredColumn(ColumnRef.create("c1")); - req.addRequiredColumn(ColumnRef.create("c2")); - req.setStartTs(1); - req.resolve(); - - List keyRanges = - ImmutableList.of( - createByteStringRange( - ByteString.copyFromUtf8("key1"), ByteString.copyFromUtf8("key4"))); - - pdServer.addGetRegionResp( - GrpcUtils.makeGetRegionResponse(pdServer.getClusterId(), region.getMeta())); - pdServer.addGetStoreResp(GrpcUtils.makeGetStoreResponse(pdServer.getClusterId(), store)); - server.putError("key1", KVMockServer.STALE_EPOCH); - CodecDataOutput cdo = new CodecDataOutput(); - IntegerCodec.writeLongFully(cdo, 666, false); - BytesCodec.writeBytesFully(cdo, "value1".getBytes()); - server.put("key1", cdo.toByteString()); - List tasks = ImmutableList.of(RegionTask.newInstance(region, store, keyRanges)); - CoprocessIterator iter = CoprocessIterator.getRowIterator(req, tasks, session); - assert iter.hasNext(); - Row r = iter.next(); - SchemaInfer infer = SchemaInfer.create(req); - assertEquals(r.get(0, infer.getType(0)), 666L); - assertEquals(r.get(1, infer.getType(1)), "value1"); - } -} diff --git a/src/test/java/org/tikv/predicates/IndexMatcherTest.java b/src/test/java/org/tikv/predicates/IndexMatcherTest.java deleted file mode 100644 index cc7dd67ad2..0000000000 --- a/src/test/java/org/tikv/predicates/IndexMatcherTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.predicates; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.tikv.expression.ComparisonBinaryExpression.equal; -import static org.tikv.expression.ComparisonBinaryExpression.greaterEqual; -import static org.tikv.expression.ComparisonBinaryExpression.lessEqual; -import static org.tikv.expression.ComparisonBinaryExpression.lessThan; -import static org.tikv.expression.LogicalBinaryExpression.and; -import static org.tikv.expression.LogicalBinaryExpression.or; - -import com.google.common.collect.ImmutableList; -import org.junit.Test; -import org.tikv.expression.ColumnRef; -import org.tikv.expression.Constant; -import org.tikv.expression.Expression; -import org.tikv.expression.visitor.IndexMatcher; -import org.tikv.meta.MetaUtils; -import org.tikv.meta.TiIndexColumn; -import org.tikv.meta.TiIndexInfo; -import org.tikv.meta.TiTableInfo; -import org.tikv.types.IntegerType; -import org.tikv.types.StringType; - -public class IndexMatcherTest { - private static TiTableInfo createTable() { - return new MetaUtils.TableBuilder() - .name("testTable") - .addColumn("c1", IntegerType.INT, true) - .addColumn("c2", StringType.VARCHAR) - .addColumn("c3", StringType.VARCHAR) - .addColumn("c4", IntegerType.INT) - .appendIndex("testIndex", ImmutableList.of("c1", "c2"), false) - .build(); - } - - @Test - public void matchOnlyEq() throws Exception { - TiTableInfo table = createTable(); - TiIndexInfo index = table.getIndices().get(0); - TiIndexColumn col = index.getIndexColumns().get(0); - IndexMatcher matcher = IndexMatcher.equalOnlyMatcher(col); - Constant c0 = Constant.create(0, IntegerType.INT); - Constant c1 = Constant.create(1, IntegerType.INT); - Constant c2 = Constant.create(2, IntegerType.INT); - ColumnRef col1 = ColumnRef.create("c1", table); - ColumnRef col2 = ColumnRef.create("c2", table); - - // index col = c1, long - Expression cond = equal(col1, c1); - assertTrue(matcher.match(cond)); - - cond = equal(c1, col1); - assertTrue(matcher.match(cond)); - - cond = equal(col2, col1); - assertFalse(matcher.match(cond)); - - cond = equal(c1, c1); - assertFalse(matcher.match(cond)); - - cond = and(equal(c1, col1), equal(col1, c2)); - assertFalse(matcher.match(cond)); - - cond = or(equal(c1, col1), equal(col1, c2)); - assertTrue(matcher.match(cond)); - - cond = lessEqual(c0, col1); - assertFalse(matcher.match(cond)); - } - - @Test - public void matchAll() throws Exception { - TiTableInfo table = createTable(); - TiIndexInfo index = table.getIndices().get(0); - TiIndexColumn col = index.getIndexColumns().get(0); - IndexMatcher matcher = IndexMatcher.matcher(col); - Constant c0 = Constant.create(0, IntegerType.INT); - Constant c1 = Constant.create(1, IntegerType.INT); - Constant c2 = Constant.create(2, IntegerType.INT); - ColumnRef col1 = ColumnRef.create("c1", table); - ColumnRef col2 = ColumnRef.create("c2", table); - - // index col = c1, long - Expression cond = lessEqual(col1, c1); - assertTrue(matcher.match(cond)); - - cond = greaterEqual(c1, col1); - assertTrue(matcher.match(cond)); - - cond = lessThan(ColumnRef.create("c2", table), ColumnRef.create("c1", table)); - assertFalse(matcher.match(cond)); - - cond = lessThan(c1, c1); - assertFalse(matcher.match(cond)); - - cond = and(lessThan(c1, col1), lessThan(col1, c2)); - assertTrue(matcher.match(cond)); - - cond = or(lessThan(c1, col1), lessThan(col1, c2)); - assertTrue(matcher.match(cond)); - } -} diff --git a/src/test/java/org/tikv/predicates/PredicateUtilsTest.java b/src/test/java/org/tikv/predicates/PredicateUtilsTest.java deleted file mode 100644 index 14f2f8ab2e..0000000000 --- a/src/test/java/org/tikv/predicates/PredicateUtilsTest.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.predicates; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.tikv.expression.ArithmeticBinaryExpression.*; -import static org.tikv.expression.ComparisonBinaryExpression.equal; -import static org.tikv.expression.ComparisonBinaryExpression.notEqual; -import static org.tikv.expression.LogicalBinaryExpression.and; -import static org.tikv.expression.LogicalBinaryExpression.or; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Range; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import org.junit.Test; -import org.tikv.expression.ColumnRef; -import org.tikv.expression.Constant; -import org.tikv.expression.Expression; -import org.tikv.key.CompoundKey; -import org.tikv.key.Key; -import org.tikv.key.TypedKey; -import org.tikv.meta.MetaUtils; -import org.tikv.meta.TiTableInfo; -import org.tikv.types.IntegerType; -import org.tikv.types.StringType; - -public class PredicateUtilsTest { - private static TiTableInfo createTable() { - return new MetaUtils.TableBuilder() - .name("testTable") - .addColumn("c1", IntegerType.INT, true) - .addColumn("c2", StringType.VARCHAR) - .addColumn("c3", StringType.VARCHAR) - .addColumn("c4", IntegerType.INT) - .addColumn("c5", IntegerType.INT) - .appendIndex("testIndex", ImmutableList.of("c1", "c2"), false) - .build(); - } - - @Test - public void mergeCNFExpressionsTest() throws Exception { - Constant c1 = Constant.create(1, IntegerType.INT); - Constant c2 = Constant.create(2, IntegerType.INT); - Constant c3 = Constant.create(3, IntegerType.INT); - Constant c4 = Constant.create(4, IntegerType.INT); - Constant c5 = Constant.create(5, IntegerType.INT); - List exprs = ImmutableList.of(c1, c2, c3, c4, c5); - - Expression res = and(c1, and(c2, and(c3, and(c4, c5)))); - assertEquals(res, PredicateUtils.mergeCNFExpressions(exprs)); - } - - @Test - public void extractColumnRefFromExpressionTest() { - TiTableInfo table = createTable(); - Constant c1 = Constant.create(1, IntegerType.INT); - Constant c2 = Constant.create(2, IntegerType.INT); - ColumnRef col1 = ColumnRef.create("c1", table); - ColumnRef col2 = ColumnRef.create("c2", table); - ColumnRef col3 = ColumnRef.create("c3", table); - ColumnRef col4 = ColumnRef.create("c4", table); - ColumnRef col5 = ColumnRef.create("c5", table); - Set baseline = ImmutableSet.of(col1, col2, col3, col4, col5); - - Expression expression = - and( - c1, - and(c2, and(col1, and(divide(col4, and(plus(col1, c1), minus(col2, col5))), col3)))); - Set columns = PredicateUtils.extractColumnRefFromExpression(expression); - assertEquals(baseline, columns); - } - - @Test - public void expressionToIndexRangesTest() { - TiTableInfo table = createTable(); - ColumnRef col1 = ColumnRef.create("c1", table); - ColumnRef col4 = ColumnRef.create("c4", table); - ColumnRef col5 = ColumnRef.create("c5", table); - Constant c1 = Constant.create(1, IntegerType.INT); - Constant c2 = Constant.create(2, IntegerType.INT); - Constant c3 = Constant.create(3, IntegerType.INT); - Constant c4 = Constant.create(4, IntegerType.INT); - TypedKey key1 = TypedKey.toTypedKey(1, IntegerType.INT); - TypedKey key2 = TypedKey.toTypedKey(2, IntegerType.INT); - TypedKey key3 = TypedKey.toTypedKey(3, IntegerType.INT); - TypedKey key4 = TypedKey.toTypedKey(4, IntegerType.INT); - - Expression predicate1 = or(or(equal(c1, col1), equal(col1, c2)), equal(col1, c1)); - Expression predicate2 = or(equal(c3, col4), equal(c4, col4)); - Expression rangePredicate = notEqual(col5, c1); - List indexRanges = - PredicateUtils.expressionToIndexRanges( - ImmutableList.of(predicate1, predicate2), Optional.of(rangePredicate), table, null); - assertEquals(8, indexRanges.size()); - Key indexKey1 = CompoundKey.concat(key1, key3); - Key indexKey2 = CompoundKey.concat(key1, key4); - Key indexKey3 = CompoundKey.concat(key2, key3); - Key indexKey4 = CompoundKey.concat(key2, key4); - - Range baselineRange1 = Range.lessThan(key1); - Range baselineRange2 = Range.greaterThan(key1); - - Set baselineKeys = ImmutableSet.of(indexKey1, indexKey2, indexKey3, indexKey4); - Set> baselineRanges = ImmutableSet.of(baselineRange1, baselineRange2); - for (IndexRange range : indexRanges) { - assertTrue(baselineKeys.contains(range.getAccessKey())); - assertTrue(baselineRanges.contains(range.getRange())); - } - } -} diff --git a/src/test/java/org/tikv/predicates/ScanAnalyzerTest.java b/src/test/java/org/tikv/predicates/ScanAnalyzerTest.java deleted file mode 100644 index 144e626843..0000000000 --- a/src/test/java/org/tikv/predicates/ScanAnalyzerTest.java +++ /dev/null @@ -1,386 +0,0 @@ -/* - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.tikv.predicates; - -import static java.util.Objects.requireNonNull; -import static org.junit.Assert.*; -import static org.tikv.expression.ComparisonBinaryExpression.*; -import static org.tikv.predicates.PredicateUtils.expressionToIndexRanges; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.protobuf.ByteString; -import java.util.*; -import org.junit.Test; -import org.tikv.expression.ColumnRef; -import org.tikv.expression.Constant; -import org.tikv.expression.Expression; -import org.tikv.key.RowKey; -import org.tikv.kvproto.Coprocessor; -import org.tikv.meta.*; -import org.tikv.meta.TiColumnInfo.InternalTypeHolder; -import org.tikv.types.*; - -public class ScanAnalyzerTest { - private static TiTableInfo createTable() { - return createTableWithIndex(1, 1); - } - - private static TiTableInfo createTableWithIndex(long tableId, long indexId) { - return new MetaUtils.TableBuilder() - .name("testTable") - .addColumn("c1", IntegerType.INT, true) - .addColumn("c2", StringType.VARCHAR) - .addColumn("c3", StringType.VARCHAR) - .addColumn("c4", IntegerType.TINYINT) - .tableId(tableId) - .appendIndex(indexId, "testIndex", ImmutableList.of("c1", "c2", "c3"), false) - .build(); - } - - private static TiTableInfo createTableWithPrefix() { - InternalTypeHolder holder = - new InternalTypeHolder( - MySQLType.TypeVarchar.getTypeCode(), - 0, - 3, // indicating a prefix type - 0, - "", - "", - "", - "", - ImmutableList.of()); - - DataType typePrefix = DataTypeFactory.of(holder); - return new MetaUtils.TableBuilder() - .name("testTable") - .addColumn("c1", IntegerType.INT, true) - .addColumn("c2", typePrefix) - .addColumn("c3", StringType.VARCHAR) - .addColumn("c4", IntegerType.INT) - .appendIndex("testIndex", ImmutableList.of("c1", "c2", "c3"), false) - .setPkHandle(true) - .build(); - } - - @Test - public void buildTableScanKeyRangeTest() throws Exception { - TiTableInfo table = createTableWithIndex(6, 5); - TiIndexInfo pkIndex = TiIndexInfo.generateFakePrimaryKeyIndex(table); - - Expression eq1 = lessThan(ColumnRef.create("c1", table), Constant.create(3, IntegerType.INT)); - - List exprs = ImmutableList.of(eq1); - - ScanSpec result = ScanAnalyzer.extractConditions(exprs, table, pkIndex); - List irs = - expressionToIndexRanges( - result.getPointPredicates(), result.getRangePredicate(), table, pkIndex); - - ScanAnalyzer scanAnalyzer = new ScanAnalyzer(); - - List keyRanges = scanAnalyzer.buildTableScanKeyRange(table, irs); - - assertEquals(keyRanges.size(), 1); - - Coprocessor.KeyRange keyRange = keyRanges.get(0); - - assertEquals( - ByteString.copyFrom( - new byte[] {116, -128, 0, 0, 0, 0, 0, 0, 6, 95, 114, 0, 0, 0, 0, 0, 0, 0, 0}), - keyRange.getStart()); - assertEquals( - ByteString.copyFrom( - new byte[] {116, -128, 0, 0, 0, 0, 0, 0, 6, 95, 115, 0, 0, 0, 0, 0, 0, 0, 0}), - keyRange.getEnd()); - } - - @Test - public void buildIndexScanKeyRangeTest() throws Exception { - TiTableInfo table = createTableWithIndex(6, 5); - TiIndexInfo index = table.getIndices().get(0); - - Expression eq1 = equal(ColumnRef.create("c1", table), Constant.create(0)); - Expression eq2 = lessEqual(ColumnRef.create("c2", table), Constant.create("wtf")); - - List exprs = ImmutableList.of(eq1); - - ScanSpec result = ScanAnalyzer.extractConditions(exprs, table, index); - List irs = - expressionToIndexRanges( - result.getPointPredicates(), result.getRangePredicate(), table, index); - - ScanAnalyzer scanAnalyzer = new ScanAnalyzer(); - - List keyRanges = scanAnalyzer.buildIndexScanKeyRange(table, index, irs); - - assertEquals(keyRanges.size(), 1); - - Coprocessor.KeyRange keyRange = keyRanges.get(0); - - assertEquals( - ByteString.copyFrom( - new byte[] { - 116, -128, 0, 0, 0, 0, 0, 0, 6, 95, 105, -128, 0, 0, 0, 0, 0, 0, 5, 3, -128, 0, 0, 0, - 0, 0, 0, 0 - }), - keyRange.getStart()); - assertEquals( - ByteString.copyFrom( - new byte[] { - 116, -128, 0, 0, 0, 0, 0, 0, 6, 95, 105, -128, 0, 0, 0, 0, 0, 0, 5, 3, -128, 0, 0, 0, - 0, 0, 0, 1 - }), - keyRange.getEnd()); - - exprs = ImmutableList.of(eq1, eq2); - result = ScanAnalyzer.extractConditions(exprs, table, index); - - irs = - expressionToIndexRanges( - result.getPointPredicates(), result.getRangePredicate(), table, index); - - keyRanges = scanAnalyzer.buildIndexScanKeyRange(table, index, irs); - - assertEquals(keyRanges.size(), 1); - - keyRange = keyRanges.get(0); - - assertEquals( - ByteString.copyFrom( - new byte[] { - 116, -128, 0, 0, 0, 0, 0, 0, 6, 95, 105, -128, 0, 0, 0, 0, 0, 0, 5, 3, -128, 0, 0, 0, - 0, 0, 0, 0, 0 - }), - keyRange.getStart()); - assertEquals( - ByteString.copyFrom( - new byte[] { - 116, -128, 0, 0, 0, 0, 0, 0, 6, 95, 105, -128, 0, 0, 0, 0, 0, 0, 5, 3, -128, 0, 0, 0, - 0, 0, 0, 0, 1, 119, 116, 102, 0, 0, 0, 0, 0, -5 - }), - keyRange.getEnd()); - } - - @Test - public void extractConditionsTest() throws Exception { - TiTableInfo table = createTable(); - TiIndexInfo index = table.getIndices().get(0); - - Expression eq1 = equal(ColumnRef.create("c1", table), Constant.create(0, IntegerType.INT)); - Expression eq2 = - equal(ColumnRef.create("c2", table), Constant.create("test", StringType.VARCHAR)); - Expression le1 = - lessEqual(ColumnRef.create("c3", table), Constant.create("fxxx", StringType.VARCHAR)); - // Last one should be pushed back - Expression eq3 = - equal(ColumnRef.create("c4", table), Constant.create("fxxx", StringType.VARCHAR)); - - List exprs = ImmutableList.of(eq1, eq2, le1, eq3); - - ScanSpec result = ScanAnalyzer.extractConditions(exprs, table, index); - assertEquals(1, result.getResidualPredicates().size()); - assertEquals(eq3, result.getResidualPredicates().toArray()[0]); - - assertEquals(2, result.getPointPredicates().size()); - assertEquals(eq1, result.getPointPredicates().get(0)); - assertEquals(eq2, result.getPointPredicates().get(1)); - - assertTrue(result.getRangePredicate().isPresent()); - assertEquals(le1, result.getRangePredicate().get()); - } - - @Test - public void extractConditionsWithPrefixTest() throws Exception { - TiTableInfo table = createTableWithPrefix(); - TiIndexInfo index = table.getIndices().get(0); - - Expression eq1 = equal(ColumnRef.create("c1", table), Constant.create(0, IntegerType.INT)); - Expression eq2 = - equal(ColumnRef.create("c2", table), Constant.create("test", StringType.VARCHAR)); - Expression le1 = - lessEqual(ColumnRef.create("c3", table), Constant.create("fxxx", StringType.VARCHAR)); - // Last one should be pushed back - Expression eq3 = - equal(ColumnRef.create("c4", table), Constant.create("fxxx", StringType.VARCHAR)); - - List exprs = ImmutableList.of(eq1, eq2, le1, eq3); - Set baselineSet = ImmutableSet.of(eq2, le1, eq3); - - ScanSpec result = ScanAnalyzer.extractConditions(exprs, table, index); - // 3 remains since c2 condition pushed back as well - assertEquals(baselineSet, result.getResidualPredicates()); - - assertEquals(2, result.getPointPredicates().size()); - assertEquals(eq1, result.getPointPredicates().get(0)); - assertEquals(eq2, result.getPointPredicates().get(1)); - - assertFalse(result.getRangePredicate().isPresent()); - } - - @Test - public void extractConditionsWithPrimaryKeyTest() throws Exception { - TiTableInfo table = createTableWithPrefix(); - TiIndexInfo index = TiIndexInfo.generateFakePrimaryKeyIndex(table); - requireNonNull(index); - assertEquals(1, index.getIndexColumns().size()); - assertEquals("c1", index.getIndexColumns().get(0).getName()); - - Expression eq1 = equal(ColumnRef.create("c1", table), Constant.create(0, IntegerType.INT)); - Expression eq2 = - equal(ColumnRef.create("c2", table), Constant.create("test", StringType.VARCHAR)); - Expression le1 = - lessEqual(ColumnRef.create("c3", table), Constant.create("fxxx", StringType.VARCHAR)); - // Last one should be pushed back - Expression eq3 = - equal(ColumnRef.create("c4", table), Constant.create("fxxx", StringType.VARCHAR)); - - List exprs = ImmutableList.of(eq1, eq2, le1, eq3); - - ScanSpec result = ScanAnalyzer.extractConditions(exprs, table, index); - - Set baselineSet = ImmutableSet.of(eq2, le1, eq3); - // 3 remains since c2 condition pushed back as well - assertEquals(baselineSet, result.getResidualPredicates()); - - assertEquals(1, result.getPointPredicates().size()); - assertEquals(eq1, result.getPointPredicates().get(0)); - - assertFalse(result.getRangePredicate().isPresent()); - } - - @Test - public void testKeyRangeGenWithNoFilterTest() throws Exception { - TiTableInfo table = createTableWithPrefix(); - TiIndexInfo index = TiIndexInfo.generateFakePrimaryKeyIndex(table); - ScanAnalyzer scanBuilder = new ScanAnalyzer(); - ScanAnalyzer.ScanPlan scanPlan = - scanBuilder.buildScan(ImmutableList.of(), ImmutableList.of(), index, table, null); - - ByteString startKey = RowKey.toRowKey(table.getId(), Long.MIN_VALUE).toByteString(); - ByteString endKey = RowKey.createBeyondMax(table.getId()).toByteString(); - - assertEquals(1, scanPlan.getKeyRanges().size()); - assertEquals(startKey, scanPlan.getKeyRanges().get(0).getStart()); - assertEquals(endKey, scanPlan.getKeyRanges().get(0).getEnd()); - } - - @Test - public void TestCoveringIndex() throws Exception { - InternalTypeHolder holder = - new InternalTypeHolder( - MySQLType.TypeVarchar.getTypeCode(), - 0, - 3, // indicating a prefix type - 0, - "", - "", - "", - "", - ImmutableList.of()); - - Map dataTypeMap = new HashMap<>(); - dataTypeMap.put("id", IntegerType.INT); - dataTypeMap.put("a", IntegerType.INT); - dataTypeMap.put("b", IntegerType.INT); - dataTypeMap.put("c", IntegerType.INT); - dataTypeMap.put("holder", DataTypeFactory.of(holder)); - - Map offsetMap = new HashMap<>(); - offsetMap.put("id", 0); - offsetMap.put("a", 1); - offsetMap.put("b", 2); - offsetMap.put("c", 3); - offsetMap.put("holder", 4); - - class test { - private String[] columnNames; - private String[] indexNames; - private int[] indexLens; - private boolean isCovering; - - private test(String[] col, String[] idx, int[] idxLen, boolean result) { - columnNames = col; - indexNames = idx; - indexLens = idxLen; - isCovering = result; - } - - private String[] getColumnNames() { - return columnNames; - } - - private String[] getIndexNames() { - return indexNames; - } - - private int[] getIndexLens() { - return indexLens; - } - } - - final test[] tests = { - new test(new String[] {"a"}, new String[] {"a"}, new int[] {-1}, true), - new test(new String[] {"a"}, new String[] {"a", "b"}, new int[] {-1, -1}, true), - new test(new String[] {"a", "b"}, new String[] {"b", "a"}, new int[] {-1, -1}, true), - new test(new String[] {"a", "b"}, new String[] {"b", "c"}, new int[] {-1, -1}, false), - new test( - new String[] {"holder", "b"}, new String[] {"holder", "b"}, new int[] {50, -1}, false), - new test(new String[] {"a", "b"}, new String[] {"a", "c"}, new int[] {-1, -1}, false), - new test(new String[] {"id", "a"}, new String[] {"a", "b"}, new int[] {-1, -1}, true) - }; - - ScanAnalyzer scanBuilder = new ScanAnalyzer(); - - for (test t : tests) { - List columns = new ArrayList<>(); - List indexCols = new ArrayList<>(); - boolean pkIsHandle = false; - for (int i = 0; i < t.getColumnNames().length; i++) { - String colName = t.getColumnNames()[i]; - if (colName.equals("id")) { - pkIsHandle = true; - } - columns.add( - new TiColumnInfo( - offsetMap.get(colName), - colName, - i, - dataTypeMap.get(colName), - colName.equals("id"))); - } - for (int i = 0; i < t.getIndexNames().length; i++) { - String idxName = t.getIndexNames()[i]; - int idxLen = t.getIndexLens()[i]; - indexCols.add(new TiIndexColumn(CIStr.newCIStr(idxName), offsetMap.get(idxName), idxLen)); - } - TiIndexInfo indexInfo = - new TiIndexInfo( - 1, - CIStr.newCIStr("test_idx"), - CIStr.newCIStr("testTable"), - ImmutableList.copyOf(indexCols), - false, - false, - SchemaState.StatePublic.getStateCode(), - "Test Index", - IndexType.IndexTypeBtree.getTypeCode(), - false); - boolean isCovering = scanBuilder.isCoveringIndex(columns, indexInfo, pkIsHandle); - assertEquals(isCovering, t.isCovering); - } - } -} diff --git a/src/test/java/org/tikv/txn/LockResolverTest.java b/src/test/java/org/tikv/txn/LockResolverTest.java index 34888a38e8..8c4e358ef8 100644 --- a/src/test/java/org/tikv/txn/LockResolverTest.java +++ b/src/test/java/org/tikv/txn/LockResolverTest.java @@ -273,27 +273,6 @@ public class LockResolverTest { public RetryException() {} } - private SelectResponse coprocess( - RegionStoreClient client, DAGRequest request, List ranges) - throws RetryException { - BackOffer backOffer = defaultBackOff(); - Queue responseQueue = new ArrayDeque<>(); - - if (client.coprocess(backOffer, request, ranges, responseQueue) != null) { - throw new RetryException(); - } - - List resultChunk = new ArrayList<>(); - while (!responseQueue.isEmpty()) { - SelectResponse response = responseQueue.poll(); - if (response != null) { - resultChunk.addAll(response.getChunksList()); - } - } - - return SelectResponse.newBuilder().addAllChunks(resultChunk).build(); - } - @Before public void setUp() throws Exception { TiConfiguration conf = TiConfiguration.createDefault("127.0.0.1:2379"); diff --git a/src/test/java/org/tikv/types/ConverterTest.java b/src/test/java/org/tikv/types/ConverterTest.java deleted file mode 100644 index f0b2e344af..0000000000 --- a/src/test/java/org/tikv/types/ConverterTest.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.tikv.types; - -import org.junit.Test; - -public class ConverterTest { - - @Test - public void convertToLongTest() throws Exception {} - - @Test - public void convertToDoubleTest() throws Exception {} - - @Test - public void convertToStringTest() throws Exception {} - - @Test - public void convertToBytesTest() throws Exception {} - - @Test - public void getLocalTimezoneTest() throws Exception {} - - @Test - public void convertToDateTimeTest() throws Exception {} - - @Test - public void convertToBigDecimalTest() throws Exception {} -} diff --git a/src/test/java/org/tikv/types/DataTypeFactoryTest.java b/src/test/java/org/tikv/types/DataTypeFactoryTest.java deleted file mode 100644 index 4146b41666..0000000000 --- a/src/test/java/org/tikv/types/DataTypeFactoryTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import static org.junit.Assert.assertEquals; - -import com.google.common.collect.ImmutableList; -import org.junit.Test; -import org.tikv.meta.TiColumnInfo.InternalTypeHolder; - -public class DataTypeFactoryTest { - - private static InternalTypeHolder createHolder(MySQLType type) { - return new InternalTypeHolder(type.getTypeCode(), 0, 0, 0, "", "", "", "", ImmutableList.of()); - } - - private void mappingTest(MySQLType type, Class cls) { - InternalTypeHolder holder = createHolder(type); - DataType dataType = DataTypeFactory.of(holder); - assertEquals(type, dataType.getType()); - assertEquals(cls, dataType.getClass()); - } - - @Test - public void of() throws Exception { - mappingTest(MySQLType.TypeBit, BitType.class); - mappingTest(MySQLType.TypeLong, IntegerType.class); - mappingTest(MySQLType.TypeTiny, IntegerType.class); - mappingTest(MySQLType.TypeVarchar, StringType.class); - mappingTest(MySQLType.TypeDate, DateType.class); - } -} diff --git a/src/test/java/org/tikv/types/DecimalTypeTest.java b/src/test/java/org/tikv/types/DecimalTypeTest.java deleted file mode 100644 index 2f90118c16..0000000000 --- a/src/test/java/org/tikv/types/DecimalTypeTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import static org.junit.Assert.assertEquals; - -import java.math.BigDecimal; -import org.junit.Test; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.types.DataType.EncodeType; - -public class DecimalTypeTest { - @Test - public void encodeTest() throws Exception { - DataType type = DecimalType.DECIMAL; - BigDecimal originalVal = BigDecimal.valueOf(6.66); - byte[] encodedKey = encode(originalVal, EncodeType.KEY, type); - Object val = decode(encodedKey, type); - assertEquals(originalVal, val); - } - - private static byte[] encode(Object val, EncodeType encodeType, DataType type) { - CodecDataOutput cdo = new CodecDataOutput(); - type.encode(cdo, encodeType, val); - return cdo.toBytes(); - } - - private static Object decode(byte[] val, DataType type) { - return type.decode(new CodecDataInput(val)); - } -} diff --git a/src/test/java/org/tikv/types/IntegerTypeTest.java b/src/test/java/org/tikv/types/IntegerTypeTest.java deleted file mode 100644 index 970aa72a06..0000000000 --- a/src/test/java/org/tikv/types/IntegerTypeTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import static org.junit.Assert.assertEquals; - -import org.junit.Test; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.types.DataType.EncodeType; - -public class IntegerTypeTest { - @Test - public void encodeTest() throws Exception { - DataType type = IntegerType.INT; - long originalVal = 666; - byte[] encodedKey = encode(originalVal, EncodeType.KEY, type); - Object val = decode(encodedKey, type); - assertEquals(originalVal, (long) val); - - encodedKey = encode(null, EncodeType.KEY, type); - val = decode(encodedKey, type); - assertEquals(null, val); - } - - private static byte[] encode(Object val, EncodeType encodeType, DataType type) { - CodecDataOutput cdo = new CodecDataOutput(); - type.encode(cdo, encodeType, val); - return cdo.toBytes(); - } - - private static Object decode(byte[] val, DataType type) { - return type.decode(new CodecDataInput(val)); - } -} diff --git a/src/test/java/org/tikv/types/RealTypeTest.java b/src/test/java/org/tikv/types/RealTypeTest.java deleted file mode 100644 index ba4eb4dc20..0000000000 --- a/src/test/java/org/tikv/types/RealTypeTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * - * Copyright 2017 PingCAP, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.tikv.types; - -import static org.junit.Assert.assertEquals; - -import org.junit.Test; -import org.tikv.codec.CodecDataInput; -import org.tikv.codec.CodecDataOutput; -import org.tikv.types.DataType.EncodeType; - -public class RealTypeTest { - @Test - public void encodeTest() throws Exception { - DataType type = RealType.DOUBLE; - double originalVal = 666.66; - byte[] encodedKey = encode(originalVal, EncodeType.KEY, type); - Object val = decode(encodedKey, type); - assertEquals(originalVal, (double) val, 0.01); - - encodedKey = encode(null, EncodeType.KEY, type); - val = decode(encodedKey, type); - assertEquals(null, val); - } - - private static byte[] encode(Object val, EncodeType encodeType, DataType type) { - CodecDataOutput cdo = new CodecDataOutput(); - type.encode(cdo, encodeType, val); - return cdo.toBytes(); - } - - private static Object decode(byte[] val, DataType type) { - return type.decode(new CodecDataInput(val)); - } -} diff --git a/src/test/java/org/tikv/util/RangeSplitterTest.java b/src/test/java/org/tikv/util/RangeSplitterTest.java deleted file mode 100644 index 139a992c50..0000000000 --- a/src/test/java/org/tikv/util/RangeSplitterTest.java +++ /dev/null @@ -1,306 +0,0 @@ -package org.tikv.util; - -import static org.junit.Assert.assertEquals; -import static org.tikv.GrpcUtils.encodeKey; - -import com.google.common.collect.ImmutableList; -import com.google.protobuf.ByteString; -import gnu.trove.list.array.TLongArrayList; -import gnu.trove.map.hash.TLongObjectHashMap; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.junit.Test; -import org.tikv.codec.Codec.IntegerCodec; -import org.tikv.codec.CodecDataOutput; -import org.tikv.key.Key; -import org.tikv.key.RowKey; -import org.tikv.key.RowKey.DecodeResult.Status; -import org.tikv.kvproto.Coprocessor.KeyRange; -import org.tikv.kvproto.Kvrpcpb.CommandPri; -import org.tikv.kvproto.Kvrpcpb.IsolationLevel; -import org.tikv.kvproto.Metapb; -import org.tikv.kvproto.Metapb.Peer; -import org.tikv.region.RegionManager; -import org.tikv.region.TiRegion; - -public class RangeSplitterTest { - static class MockRegionManager extends RegionManager { - private final Map mockRegionMap; - - public MockRegionManager(List ranges) { - super(null); - mockRegionMap = - ranges.stream().collect(Collectors.toMap(kr -> kr, kr -> region(ranges.indexOf(kr), kr))); - } - - @Override - public TiRegion getRegionById(long regionId) { - return mockRegionMap - .entrySet() - .stream() - .filter(e -> e.getValue().getId() == regionId) - .findFirst() - .get() - .getValue(); - } - - @Override - public Pair getRegionStorePairByRegionId(long id) { - Map.Entry entry = - mockRegionMap - .entrySet() - .stream() - .filter(e -> e.getValue().getId() == id) - .findFirst() - .get(); - return Pair.create( - entry.getValue(), Metapb.Store.newBuilder().setId(entry.getValue().getId()).build()); - } - - @Override - public Pair getRegionStorePairByKey(ByteString key) { - for (Map.Entry entry : mockRegionMap.entrySet()) { - KeyRange range = entry.getKey(); - if (KeyRangeUtils.makeRange(range.getStart(), range.getEnd()).contains(Key.toRawKey(key))) { - TiRegion region = entry.getValue(); - return Pair.create(region, Metapb.Store.newBuilder().setId(region.getId()).build()); - } - } - return null; - } - } - - private static KeyRange keyRange(Long s, Long e) { - ByteString sKey = ByteString.EMPTY; - ByteString eKey = ByteString.EMPTY; - if (s != null) { - CodecDataOutput cdo = new CodecDataOutput(); - IntegerCodec.writeLongFully(cdo, s, true); - sKey = cdo.toByteString(); - } - - if (e != null) { - CodecDataOutput cdo = new CodecDataOutput(); - IntegerCodec.writeLongFully(cdo, e, true); - eKey = cdo.toByteString(); - } - - return KeyRange.newBuilder().setStart(sKey).setEnd(eKey).build(); - } - - private static ByteString shiftByStatus(ByteString v, Status s) { - switch (s) { - case EQUAL: - return v; - case LESS: - return v.substring(0, v.size() - 1); - case GREATER: - return v.concat(ByteString.copyFrom(new byte[] {1, 0})); - default: - throw new IllegalArgumentException("Only EQUAL,LESS,GREATER allowed"); - } - } - - private static KeyRange keyRangeByHandle(long tableId, Long s, Status ss, Long e, Status es) { - ByteString sKey = shiftByStatus(handleToByteString(tableId, s), ss); - ByteString eKey = shiftByStatus(handleToByteString(tableId, e), es); - - return KeyRange.newBuilder().setStart(sKey).setEnd(eKey).build(); - } - - private static ByteString handleToByteString(long tableId, Long k) { - if (k != null) { - return RowKey.toRowKey(tableId, k).toByteString(); - } - return ByteString.EMPTY; - } - - private static KeyRange keyRangeByHandle(long tableId, Long s, Long e) { - return keyRangeByHandle(tableId, s, Status.EQUAL, e, Status.EQUAL); - } - - private static TiRegion region(long id, KeyRange range) { - return new TiRegion( - Metapb.Region.newBuilder() - .setId(id) - .setStartKey(encodeKey(range.getStart().toByteArray())) - .setEndKey(encodeKey(range.getEnd().toByteArray())) - .addPeers(Peer.getDefaultInstance()) - .build(), - null, - IsolationLevel.RC, - CommandPri.Low); - } - - @Test - public void splitRangeByRegionTest() throws Exception { - MockRegionManager mgr = - new MockRegionManager( - ImmutableList.of(keyRange(null, 30L), keyRange(30L, 50L), keyRange(50L, null))); - RangeSplitter s = RangeSplitter.newSplitter(mgr); - List tasks = - s.splitRangeByRegion( - ImmutableList.of( - keyRange(0L, 40L), keyRange(41L, 42L), keyRange(45L, 50L), keyRange(70L, 1000L))); - - assertEquals(tasks.get(0).getRegion().getId(), 0); - assertEquals(tasks.get(0).getRanges().size(), 1); - assertEquals(tasks.get(0).getRanges().get(0), keyRange(0L, 30L)); - - assertEquals(tasks.get(1).getRegion().getId(), 1); - assertEquals(tasks.get(1).getRanges().get(0), keyRange(30L, 40L)); - assertEquals(tasks.get(1).getRanges().get(1), keyRange(41L, 42L)); - assertEquals(tasks.get(1).getRanges().get(2), keyRange(45L, 50L)); - assertEquals(tasks.get(1).getRanges().size(), 3); - - assertEquals(tasks.get(2).getRegion().getId(), 2); - assertEquals(tasks.get(2).getRanges().size(), 1); - assertEquals(tasks.get(2).getRanges().get(0), keyRange(70L, 1000L)); - } - - @Test - public void splitAndSortHandlesByRegionTest() throws Exception { - final long tableId = 1; - TLongArrayList handles = new TLongArrayList(); - handles.add( - new long[] { - 1, - 5, - 4, - 3, - 10, - 2, - 100, - 101, - 99, - 88, - -1, - -255, - -100, - -99, - -98, - Long.MIN_VALUE, - 8960, - 8959, - 19999, - 15001 - }); - - MockRegionManager mgr = - new MockRegionManager( - ImmutableList.of( - keyRangeByHandle(tableId, null, Status.EQUAL, -100L, Status.EQUAL), - keyRangeByHandle(tableId, -100L, Status.EQUAL, 10L, Status.GREATER), - keyRangeByHandle(tableId, 10L, Status.GREATER, 50L, Status.EQUAL), - keyRangeByHandle(tableId, 50L, Status.EQUAL, 100L, Status.GREATER), - keyRangeByHandle(tableId, 100L, Status.GREATER, 9000L, Status.LESS), - keyRangeByHandle(tableId, 0x2300L /*8960*/, Status.LESS, 16000L, Status.EQUAL), - keyRangeByHandle(tableId, 16000L, Status.EQUAL, null, Status.EQUAL))); - - RangeSplitter s = RangeSplitter.newSplitter(mgr); - List tasks = - new ArrayList<>(s.splitAndSortHandlesByRegion(tableId, handles)); - tasks.sort( - (l, r) -> { - Long regionIdLeft = l.getRegion().getId(); - Long regionIdRight = r.getRegion().getId(); - return regionIdLeft.compareTo(regionIdRight); - }); - - // [-INF, -100): [Long.MIN_VALUE, Long.MIN_VALUE + 1), [-255, -254) - assertEquals(tasks.get(0).getRegion().getId(), 0); - assertEquals(tasks.get(0).getRanges().size(), 2); - assertEquals( - tasks.get(0).getRanges().get(0), - keyRangeByHandle(tableId, Long.MIN_VALUE, Long.MIN_VALUE + 1)); - assertEquals(tasks.get(0).getRanges().get(1), keyRangeByHandle(tableId, -255L, -254L)); - - // [-100, 10.x): [-100, -97), [-1, 0), [1, 6), [10, 11) - assertEquals(tasks.get(1).getRegion().getId(), 1); - assertEquals(tasks.get(1).getRanges().size(), 4); - assertEquals(tasks.get(1).getRanges().get(0), keyRangeByHandle(tableId, -100L, -97L)); - assertEquals(tasks.get(1).getRanges().get(1), keyRangeByHandle(tableId, -1L, 0L)); - assertEquals(tasks.get(1).getRanges().get(2), keyRangeByHandle(tableId, 1L, 6L)); - assertEquals(tasks.get(1).getRanges().get(3), keyRangeByHandle(tableId, 10L, 11L)); - - // [10.x, 50): empty - // [50, 100.x): [88, 89) [99, 101) - assertEquals(tasks.get(2).getRegion().getId(), 3); - assertEquals(tasks.get(2).getRanges().size(), 2); - assertEquals(tasks.get(2).getRanges().get(0), keyRangeByHandle(tableId, 88L, 89L)); - assertEquals(tasks.get(2).getRanges().get(1), keyRangeByHandle(tableId, 99L, 101L)); - - // [100.x, less than 8960): [101, 102) [8959, 8960) - assertEquals(tasks.get(3).getRegion().getId(), 4); - assertEquals(tasks.get(3).getRanges().size(), 2); - assertEquals(tasks.get(3).getRanges().get(0), keyRangeByHandle(tableId, 101L, 102L)); - assertEquals(tasks.get(3).getRanges().get(1), keyRangeByHandle(tableId, 8959L, 8960L)); - - // [less than 8960, 16000): [9000, 9001), [15001, 15002) - assertEquals(tasks.get(4).getRegion().getId(), 5); - assertEquals(tasks.get(4).getRanges().size(), 2); - assertEquals(tasks.get(4).getRanges().get(0), keyRangeByHandle(tableId, 8960L, 8961L)); - assertEquals(tasks.get(4).getRanges().get(1), keyRangeByHandle(tableId, 15001L, 15002L)); - - // [16000, INF): [19999, 20000) - assertEquals(tasks.get(5).getRegion().getId(), 6); - assertEquals(tasks.get(5).getRanges().size(), 1); - assertEquals(tasks.get(5).getRanges().get(0), keyRangeByHandle(tableId, 19999L, 20000L)); - } - - @Test - public void groupByAndSortHandlesByRegionIdTest() { - final long tableId = 1; - TLongArrayList handles = new TLongArrayList(); - handles.add( - new long[] { - 1, - 5, - 4, - 3, - 10, - 11, - 12, - 2, - 100, - 101, - 99, - 88, - -1, - -255, - -100, - -99, - -98, - Long.MIN_VALUE, - 8960, - 8959, - 19999, - 15001, - 99999999999L, - Long.MAX_VALUE - }); - - MockRegionManager mgr = - new MockRegionManager( - ImmutableList.of( - keyRangeByHandle(tableId, null, Status.EQUAL, -100L, Status.EQUAL), - keyRangeByHandle(tableId, -100L, Status.EQUAL, 10L, Status.GREATER), - keyRangeByHandle(tableId, 10L, Status.GREATER, 50L, Status.EQUAL), - keyRangeByHandle(tableId, 50L, Status.EQUAL, 100L, Status.GREATER), - keyRangeByHandle(tableId, 100L, Status.GREATER, 9000L, Status.LESS), - keyRangeByHandle(tableId, 0x2300L /*8960*/, Status.LESS, 16000L, Status.EQUAL), - keyRangeByHandle(tableId, 16000L, Status.EQUAL, null, Status.EQUAL))); - - TLongObjectHashMap result = - RangeSplitter.newSplitter(mgr).groupByAndSortHandlesByRegionId(tableId, handles); - assertEquals(2, result.get(0).size()); - assertEquals(10, result.get(1).size()); - assertEquals(2, result.get(2).size()); - assertEquals(3, result.get(3).size()); - assertEquals(2, result.get(4).size()); - assertEquals(2, result.get(5).size()); - assertEquals(3, result.get(6).size()); - } -}