rawclient: add batch scan keys (#191)

Signed-off-by: iosmanthus <myosmanthustree@gmail.com>
This commit is contained in:
Iosmanthus Teng 2021-06-09 15:47:58 +08:00 committed by GitHub
parent 3be4daf78c
commit 52f8c25d8b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 61 additions and 0 deletions

View File

@ -345,6 +345,49 @@ public class RawKVClient implements AutoCloseable {
}
}
/**
* Create a new `batch scan` request with `keyOnly` option Once resolved this request will result
* in a set of scanners over the given keys.
*
* <p>WARNING: This method is experimental. The `each_limit` parameter does not work as expected.
* It does not limit the number of results returned of each range, instead it limits the number of
* results in each region of each range. As a result, you may get more than each_limit key-value
* pairs for each range. But you should not miss any entries.
*
* @param ranges a list of ranges
* @return a set of scanners for keys over the given keys.
*/
public List<List<ByteString>> batchScanKeys(
List<Pair<ByteString, ByteString>> ranges, int eachLimit) {
return batchScan(
ranges
.stream()
.map(
range ->
ScanOption.newBuilder()
.setStartKey(range.first)
.setEndKey(range.second)
.setLimit(eachLimit)
.setKeyOnly(true)
.build())
.collect(Collectors.toList()))
.stream()
.map(kvs -> kvs.stream().map(kv -> kv.getKey()).collect(Collectors.toList()))
.collect(Collectors.toList());
}
/**
* Create a new `batch scan` request. Once resolved this request will result in a set of scanners
* over the given keys.
*
* <p>WARNING: This method is experimental. The `each_limit` parameter does not work as expected.
* It does not limit the number of results returned of each range, instead it limits the number of
* results in each region of each range. As a result, you may get more than each_limit key-value
* pairs for each range. But you should not miss any entries.
*
* @param ranges a list of `ScanOption` for each range
* @return a set of scanners over the given keys.
*/
public List<List<KvPair>> batchScan(List<ScanOption> ranges) {
String label = "client_raw_batch_scan";
Histogram.Timer requestTimer = RAW_REQUEST_LATENCY.labels(label).startTimer();

View File

@ -18,6 +18,7 @@ import org.tikv.common.codec.KeyUtils;
import org.tikv.common.exception.TiKVException;
import org.tikv.common.key.Key;
import org.tikv.common.util.FastByteComparisons;
import org.tikv.common.util.Pair;
import org.tikv.common.util.ScanOption;
import org.tikv.kvproto.Kvrpcpb;
@ -644,6 +645,11 @@ public class RawKVClientTest {
scanOptions.add(scanOption);
}
checkBatchScan(scanOptions);
checkBatchScanKeys(
scanOptions
.stream()
.map(scanOption -> Pair.create(scanOption.getStartKey(), scanOption.getEndKey()))
.collect(Collectors.toList()));
}
}
@ -801,6 +807,7 @@ public class RawKVClientTest {
}
private void checkBatchScan(List<ScanOption> scanOptions) {
logger.info("checking batch scan");
List<List<Kvrpcpb.KvPair>> result = client.batchScan(scanOptions);
int i = 0;
for (ScanOption scanOption : scanOptions) {
@ -820,6 +827,17 @@ public class RawKVClientTest {
}
}
private void checkBatchScanKeys(List<Pair<ByteString, ByteString>> ranges) {
logger.info("checking batch scan keys");
List<List<ByteString>> result = client.batchScanKeys(ranges, limit);
for (int i = 0; i < ranges.size(); i++) {
Pair<ByteString, ByteString> range = ranges.get(i);
List<ByteString> partialResult =
new ArrayList<>(data.subMap(range.first, range.second).keySet());
assert result.get(i).equals(partialResult);
}
}
private void checkDelete(ByteString key) {
client.delete(key);
checkEmpty(key);