diff --git a/pom.xml b/pom.xml
index e56c6a0613..4aa8c96893 100644
--- a/pom.xml
+++ b/pom.xml
@@ -61,6 +61,7 @@
1.8
UTF-8
UTF-8
+ 6.22.1.1
3.5.1
1.2.17
1.7.16
@@ -79,6 +80,11 @@
+
+ org.rocksdb
+ rocksdbjni
+ ${rocksdb.version}
+
org.antlr
antlr4-runtime
diff --git a/scripts/proto.sh b/scripts/proto.sh
index 8d65d7dd2f..a5e54ccd4e 100755
--- a/scripts/proto.sh
+++ b/scripts/proto.sh
@@ -14,7 +14,7 @@
# limitations under the License.
#
-kvproto_hash=2ac2a7984b2d01b96ed56fd8474f4bf80fa33c51
+kvproto_hash=465fa4c7b42e644d5aacaf79d06c75733dc12eb3
raft_rs_hash=b9891b673573fad77ebcf9bbe0969cf945841926
tipb_hash=c4d518eb1d60c21f05b028b36729e64610346dac
diff --git a/src/main/java/org/tikv/br/BackupDecoder.java b/src/main/java/org/tikv/br/BackupDecoder.java
new file mode 100644
index 0000000000..fb63d63ee4
--- /dev/null
+++ b/src/main/java/org/tikv/br/BackupDecoder.java
@@ -0,0 +1,64 @@
+/*
+ *
+ * Copyright 2021 PingCAP, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.br;
+
+import org.rocksdb.Options;
+import org.rocksdb.ReadOptions;
+import org.tikv.common.exception.SSTDecodeException;
+import org.tikv.kvproto.Brpb;
+
+public class BackupDecoder {
+ private final Brpb.BackupMeta backupMeta;
+ private final boolean ttlEnabled;
+ private final KVDecoder kvDecoder;
+
+ public BackupDecoder(Brpb.BackupMeta backupMeta) throws SSTDecodeException {
+ this.backupMeta = backupMeta;
+ this.ttlEnabled = false;
+ this.kvDecoder = initKVDecoder();
+ }
+
+ public BackupDecoder(Brpb.BackupMeta backupMeta, boolean ttlEnabled) throws SSTDecodeException {
+ this.backupMeta = backupMeta;
+ this.ttlEnabled = ttlEnabled;
+ this.kvDecoder = initKVDecoder();
+ }
+
+ private KVDecoder initKVDecoder() throws SSTDecodeException {
+ // Currently only v1 is supported.
+ // V2 will be added after https://github.com/tikv/tikv/issues/10938.
+ if (backupMeta.getIsRawKv()) {
+ // TODO: ttl_enable should be witten to BackupMeta
+ return new RawKVDecoderV1(ttlEnabled);
+ } else {
+ throw new SSTDecodeException("TxnKV is not supported yet!");
+ }
+ }
+
+ public SSTDecoder decodeSST(String sstFilePath) {
+ return decodeSST(sstFilePath, new Options(), new ReadOptions());
+ }
+
+ public SSTDecoder decodeSST(String sstFilePath, Options options, ReadOptions readOptions) {
+ return new SSTDecoder(sstFilePath, kvDecoder, options, readOptions);
+ }
+
+ public Brpb.BackupMeta getBackupMeta() {
+ return backupMeta;
+ }
+}
diff --git a/src/main/java/org/tikv/br/BackupMetaDecoder.java b/src/main/java/org/tikv/br/BackupMetaDecoder.java
new file mode 100644
index 0000000000..ce4a258fe9
--- /dev/null
+++ b/src/main/java/org/tikv/br/BackupMetaDecoder.java
@@ -0,0 +1,41 @@
+/*
+ *
+ * Copyright 2021 PingCAP, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.br;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import org.tikv.kvproto.Brpb;
+
+public class BackupMetaDecoder {
+ private final Brpb.BackupMeta backupMeta;
+
+ public BackupMetaDecoder(byte[] data) throws InvalidProtocolBufferException {
+ this.backupMeta = Brpb.BackupMeta.parseFrom(data);
+ }
+
+ public Brpb.BackupMeta getBackupMeta() {
+ return backupMeta;
+ }
+
+ public static BackupMetaDecoder parse(String backupMetaFilePath) throws IOException {
+ byte[] data = Files.readAllBytes(new File(backupMetaFilePath).toPath());
+ return new BackupMetaDecoder(data);
+ }
+}
diff --git a/src/main/java/org/tikv/br/KVDecoder.java b/src/main/java/org/tikv/br/KVDecoder.java
new file mode 100644
index 0000000000..1f0b580c1a
--- /dev/null
+++ b/src/main/java/org/tikv/br/KVDecoder.java
@@ -0,0 +1,26 @@
+/*
+ *
+ * Copyright 2021 PingCAP, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.br;
+
+import com.google.protobuf.ByteString;
+
+public interface KVDecoder {
+ ByteString decodeKey(byte[] key);
+
+ ByteString decodeValue(byte[] value);
+}
diff --git a/src/main/java/org/tikv/br/RawKVDecoderV1.java b/src/main/java/org/tikv/br/RawKVDecoderV1.java
new file mode 100644
index 0000000000..7b2e50db9b
--- /dev/null
+++ b/src/main/java/org/tikv/br/RawKVDecoderV1.java
@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2021 PingCAP, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.br;
+
+import com.google.protobuf.ByteString;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RawKVDecoderV1 implements KVDecoder {
+ private static final Logger logger = LoggerFactory.getLogger(SSTIterator.class);
+
+ private final boolean ttlEnabled;
+
+ public RawKVDecoderV1(boolean ttlEnabled) {
+ this.ttlEnabled = ttlEnabled;
+ }
+
+ @Override
+ public ByteString decodeKey(byte[] key) {
+ if (key == null || key.length == 0) {
+ logger.warn(
+ "skip Key-Value pair because key == null || key.length == 0, key = "
+ + Arrays.toString(key));
+ return null;
+ } else if (key[0] != 'z') {
+ logger.warn("skip Key-Value pair because key[0] != 'z', key = " + Arrays.toString(key));
+ return null;
+ }
+ return ByteString.copyFrom(key, 1, key.length - 1);
+ }
+
+ @Override
+ public ByteString decodeValue(byte[] value) {
+ if (!ttlEnabled) {
+ return ByteString.copyFrom(value);
+ } else {
+ return ByteString.copyFrom(value).substring(0, value.length - 8);
+ }
+ }
+}
diff --git a/src/main/java/org/tikv/br/SSTDecoder.java b/src/main/java/org/tikv/br/SSTDecoder.java
new file mode 100644
index 0000000000..1841b4e9db
--- /dev/null
+++ b/src/main/java/org/tikv/br/SSTDecoder.java
@@ -0,0 +1,93 @@
+/*
+ *
+ * Copyright 2021 PingCAP, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.br;
+
+import com.google.protobuf.ByteString;
+import java.util.Iterator;
+import org.rocksdb.Options;
+import org.rocksdb.ReadOptions;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.SstFileReader;
+import org.rocksdb.SstFileReaderIterator;
+import org.tikv.common.util.Pair;
+
+public class SSTDecoder {
+ private final String filePath;
+ private final KVDecoder kvDecoder;
+ private final Options options;
+ private final ReadOptions readOptions;
+
+ private SstFileReader sstFileReader;
+ private SstFileReaderIterator iterator;
+
+ public SSTDecoder(String sstFilePath, KVDecoder kvDecoder) {
+ this.filePath = sstFilePath;
+ this.kvDecoder = kvDecoder;
+ this.options = new Options();
+ this.readOptions = new ReadOptions();
+ }
+
+ public SSTDecoder(
+ String filePath, KVDecoder kvDecoder, Options options, ReadOptions readOptions) {
+ this.filePath = filePath;
+ this.kvDecoder = kvDecoder;
+ this.options = options;
+ this.readOptions = readOptions;
+ }
+
+ public synchronized Iterator> getIterator() throws RocksDBException {
+ if (sstFileReader != null || iterator != null) {
+ throw new RocksDBException("File already opened!");
+ }
+
+ sstFileReader = new SstFileReader(new Options());
+ sstFileReader.open(filePath);
+ iterator = sstFileReader.newIterator(new ReadOptions());
+ return new SSTIterator(iterator, kvDecoder);
+ }
+
+ public synchronized void close() {
+ try {
+ if (iterator != null) {
+ iterator.close();
+ }
+ } finally {
+ iterator = null;
+ }
+
+ try {
+ if (sstFileReader != null) {
+ sstFileReader.close();
+ }
+ } finally {
+ sstFileReader = null;
+ }
+ }
+
+ public String getFilePath() {
+ return filePath;
+ }
+
+ public Options getOptions() {
+ return options;
+ }
+
+ public ReadOptions getReadOptions() {
+ return readOptions;
+ }
+}
diff --git a/src/main/java/org/tikv/br/SSTIterator.java b/src/main/java/org/tikv/br/SSTIterator.java
new file mode 100644
index 0000000000..0b8e40c5c3
--- /dev/null
+++ b/src/main/java/org/tikv/br/SSTIterator.java
@@ -0,0 +1,64 @@
+/*
+ *
+ * Copyright 2021 PingCAP, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.br;
+
+import com.google.protobuf.ByteString;
+import java.util.Iterator;
+import org.rocksdb.SstFileReaderIterator;
+import org.tikv.common.util.Pair;
+
+public class SSTIterator implements Iterator> {
+ private final SstFileReaderIterator iterator;
+ private final KVDecoder kvDecoder;
+
+ private Pair nextPair;
+
+ public SSTIterator(SstFileReaderIterator iterator, KVDecoder kvDecoder) {
+ this.iterator = iterator;
+ this.kvDecoder = kvDecoder;
+ this.iterator.seekToFirst();
+ this.nextPair = processNext();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return nextPair != null;
+ }
+
+ @Override
+ public Pair next() {
+ Pair result = nextPair;
+ nextPair = processNext();
+ return result;
+ }
+
+ private Pair processNext() {
+ if (iterator.isValid()) {
+ ByteString key = kvDecoder.decodeKey(iterator.key());
+ ByteString value = kvDecoder.decodeValue(iterator.value());
+ iterator.next();
+ if (key != null) {
+ return Pair.create(key, value);
+ } else {
+ return processNext();
+ }
+ } else {
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/org/tikv/common/exception/SSTDecodeException.java b/src/main/java/org/tikv/common/exception/SSTDecodeException.java
new file mode 100644
index 0000000000..5d8ecfec80
--- /dev/null
+++ b/src/main/java/org/tikv/common/exception/SSTDecodeException.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2021 PingCAP, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.tikv.common.exception;
+
+public class SSTDecodeException extends RuntimeException {
+ public SSTDecodeException(Exception e) {
+ super(e);
+ }
+
+ public SSTDecodeException(String msg) {
+ super(msg);
+ }
+}
diff --git a/src/main/java/org/tikv/common/region/RegionStoreClient.java b/src/main/java/org/tikv/common/region/RegionStoreClient.java
index bfe655e912..29d3bdc64d 100644
--- a/src/main/java/org/tikv/common/region/RegionStoreClient.java
+++ b/src/main/java/org/tikv/common/region/RegionStoreClient.java
@@ -1042,7 +1042,7 @@ public class RegionStoreClient extends AbstractRegionStoreClient {
RawBatchPutRequest.newBuilder()
.setContext(makeContext(storeType))
.addAllPairs(kvPairs)
- .setTtl(ttl)
+ .addTtls(ttl)
.setForCas(atomicForCAS)
.build();
RegionErrorHandler handler =
diff --git a/src/test/java/org/tikv/br/BackupDecoderTest.java b/src/test/java/org/tikv/br/BackupDecoderTest.java
new file mode 100644
index 0000000000..c1f81de595
--- /dev/null
+++ b/src/test/java/org/tikv/br/BackupDecoderTest.java
@@ -0,0 +1,78 @@
+package org.tikv.br;
+
+import com.google.protobuf.ByteString;
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+import org.junit.Assert;
+import org.junit.Test;
+import org.rocksdb.RocksDBException;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.Brpb;
+
+public class BackupDecoderTest {
+
+ private static final int TOTAL_COUNT = 500;
+ private static final String KEY_PREFIX = "test_";
+ private static final String VALUE =
+ "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
+
+ @Test
+ public void rawKVSSTDecoderTest() throws RocksDBException, IOException {
+ String backupmetaFilePath = "src/test/resources/sst/backupmeta";
+ String sst1FilePath =
+ "src/test/resources/sst/1_2_2_7154800cc311f03afd1532e961b9a878dfbb119b104cf4daad5d0c7c0eacb502_1633919546277_default.sst";
+ String sst2FilePath =
+ "src/test/resources/sst/4_8_2_9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08_1633919546278_default.sst";
+
+ BackupMetaDecoder backupMetaDecoder = BackupMetaDecoder.parse(backupmetaFilePath);
+ Brpb.BackupMeta backupMeta = backupMetaDecoder.getBackupMeta();
+
+ BackupDecoder sstBackup = new BackupDecoder(backupMeta);
+
+ decodeSST(sstBackup, sst1FilePath);
+ decodeSST(sstBackup, sst2FilePath);
+ }
+
+ @Test
+ public void rawKVWithTTLSSTDecoderTest() throws RocksDBException, IOException {
+ String backupmetaFilePath = "src/test/resources/sst_ttl/backupmeta";
+ String sst1FilePath =
+ "src/test/resources/sst_ttl/1_2_2_7154800cc311f03afd1532e961b9a878dfbb119b104cf4daad5d0c7c0eacb502_1634199092593_default.sst";
+ String sst2FilePath =
+ "src/test/resources/sst_ttl/5_8_2_9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08_1634199092587_default.sst";
+
+ BackupMetaDecoder backupMetaDecoder = BackupMetaDecoder.parse(backupmetaFilePath);
+ Brpb.BackupMeta backupMeta = backupMetaDecoder.getBackupMeta();
+
+ BackupDecoder sstBackup = new BackupDecoder(backupMeta, true);
+
+ decodeSST(sstBackup, sst1FilePath);
+ decodeSST(sstBackup, sst2FilePath);
+ }
+
+ private void decodeSST(BackupDecoder sstBackup, String sst) throws RocksDBException {
+ String fileName = new File(sst).getName();
+ Brpb.File backupFile =
+ sstBackup
+ .getBackupMeta()
+ .getFilesList()
+ .stream()
+ .filter(a -> a.getName().equals(fileName))
+ .findFirst()
+ .get();
+ Assert.assertEquals(TOTAL_COUNT, backupFile.getTotalKvs());
+
+ SSTDecoder sstDecoder = sstBackup.decodeSST(sst);
+ Iterator> iterator = sstDecoder.getIterator();
+ int count = 0;
+ while (iterator.hasNext()) {
+ Pair pair = iterator.next();
+ Assert.assertEquals(VALUE, pair.second.toStringUtf8());
+ Assert.assertTrue(pair.first.toStringUtf8().startsWith(KEY_PREFIX));
+ count += 1;
+ }
+ sstDecoder.close();
+ Assert.assertEquals(TOTAL_COUNT, count);
+ }
+}
diff --git a/src/test/resources/sst/1_2_2_7154800cc311f03afd1532e961b9a878dfbb119b104cf4daad5d0c7c0eacb502_1633919546277_default.sst b/src/test/resources/sst/1_2_2_7154800cc311f03afd1532e961b9a878dfbb119b104cf4daad5d0c7c0eacb502_1633919546277_default.sst
new file mode 100644
index 0000000000..8ca01d4678
Binary files /dev/null and b/src/test/resources/sst/1_2_2_7154800cc311f03afd1532e961b9a878dfbb119b104cf4daad5d0c7c0eacb502_1633919546277_default.sst differ
diff --git a/src/test/resources/sst/4_8_2_9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08_1633919546278_default.sst b/src/test/resources/sst/4_8_2_9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08_1633919546278_default.sst
new file mode 100644
index 0000000000..03bcd27c11
Binary files /dev/null and b/src/test/resources/sst/4_8_2_9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08_1633919546278_default.sst differ
diff --git a/src/test/resources/sst/backupmeta b/src/test/resources/sst/backupmeta
new file mode 100644
index 0000000000..abcca0b24c
--- /dev/null
+++ b/src/test/resources/sst/backupmeta
@@ -0,0 +1,11 @@
+턘a"5.3.0-alpha"
+"
+`4_8_2_9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08_1633919546278_default.sst -婔1N7@h*2,L>c_Q?test"test_-009965169116504@HRdefaultX/"
+`1_2_2_7154800cc311f03afd1532e961b9a878dfbb119b104cf4daad5d0c7c0eacb502_1633919546277_default.sst H]f;FjLV@7ytest_-009965169116504"u@HRdefaultX0@J
+testudefaultR[]ZBR
+Release Version: v5.2.1
+Git Commit Hash: cd8fb24c5f7ebd9d479ed228bb41848bd5e97445
+Git Branch: heads/refs/tags/v5.2.1
+Go Version: go1.16.4
+UTC Build Time: 2021-09-07 16:19:11
+Race Enabled: false
\ No newline at end of file
diff --git a/src/test/resources/sst_ttl/1_2_2_7154800cc311f03afd1532e961b9a878dfbb119b104cf4daad5d0c7c0eacb502_1634199092593_default.sst b/src/test/resources/sst_ttl/1_2_2_7154800cc311f03afd1532e961b9a878dfbb119b104cf4daad5d0c7c0eacb502_1634199092593_default.sst
new file mode 100644
index 0000000000..6bf2760fa0
Binary files /dev/null and b/src/test/resources/sst_ttl/1_2_2_7154800cc311f03afd1532e961b9a878dfbb119b104cf4daad5d0c7c0eacb502_1634199092593_default.sst differ
diff --git a/src/test/resources/sst_ttl/5_8_2_9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08_1634199092587_default.sst b/src/test/resources/sst_ttl/5_8_2_9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08_1634199092587_default.sst
new file mode 100644
index 0000000000..9c582e5818
Binary files /dev/null and b/src/test/resources/sst_ttl/5_8_2_9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08_1634199092587_default.sst differ
diff --git a/src/test/resources/sst_ttl/backupmeta b/src/test/resources/sst_ttl/backupmeta
new file mode 100644
index 0000000000..978ef5f61a
--- /dev/null
+++ b/src/test/resources/sst_ttl/backupmeta
@@ -0,0 +1,11 @@
+ȗa"5.3.0-alpha"
+"
+`5_8_2_9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08_1634199092587_default.sst I~Ѷ:8~0KFtest"test_-009965169116504@HRdefaultX1"
+`1_2_2_7154800cc311f03afd1532e961b9a878dfbb119b104cf4daad5d0c7c0eacb502_1634199092593_default.sst w2l^9>6#Մލ)qar~test_-009965169116504"u@HRdefaultX1@J
+testudefaultR[]ZBR
+Release Version: v5.2.1
+Git Commit Hash: cd8fb24c5f7ebd9d479ed228bb41848bd5e97445
+Git Branch: heads/refs/tags/v5.2.1
+Go Version: go1.16.4
+UTC Build Time: 2021-09-07 16:19:11
+Race Enabled: false
\ No newline at end of file