mirror of https://github.com/tikv/client-go.git
region cache: check if the pd returned regions covers the ranges (#1377)
* add reproduce test Signed-off-by: you06 <you1474600@gmail.com> * check if the pd returned regions covers the ranges Signed-off-by: you06 <you1474600@gmail.com> * handle limit Signed-off-by: you06 <you1474600@gmail.com> * fix lint Signed-off-by: you06 <you1474600@gmail.com> * add TODO for func Signed-off-by: you06 <you1474600@gmail.com> * update tidb for integration test Signed-off-by: you06 <you1474600@gmail.com> * rename gap detection function Signed-off-by: you06 <you1474600@gmail.com> * address comment Signed-off-by: you06 <you1474600@gmail.com> * address comment Signed-off-by: you06 <you1474600@gmail.com> * add half bounded cases Signed-off-by: you06 <you1474600@gmail.com> --------- Signed-off-by: you06 <you1474600@gmail.com>
This commit is contained in:
you06
GitHub
parent
0206a3c142
commit
d73cc1ed65
|
|
@ -7,11 +7,11 @@ require (
|
|||
github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f
|
||||
github.com/pingcap/failpoint v0.0.0-20240527053858-9b3b6e34194a
|
||||
github.com/pingcap/kvproto v0.0.0-20240620063548-118a4cab53e4
|
||||
github.com/pingcap/tidb v1.1.0-beta.0.20240430081142-7481aa6d0b8b
|
||||
github.com/pingcap/tidb v1.1.0-beta.0.20240703042657-230bbc2ef5ef
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/stretchr/testify v1.9.0
|
||||
github.com/tidwall/gjson v1.14.1
|
||||
github.com/tikv/client-go/v2 v2.0.8-0.20240621090319-d4f0f4cf12a9
|
||||
github.com/tikv/client-go/v2 v2.0.8-0.20240626064248-4a72526f6c30
|
||||
github.com/tikv/pd/client v0.0.0-20240620115049-049de1761e56
|
||||
go.uber.org/goleak v1.3.0
|
||||
)
|
||||
|
|
@ -57,6 +57,7 @@ require (
|
|||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/ncw/directio v1.0.5 // indirect
|
||||
github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 // indirect
|
||||
github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef // indirect
|
||||
|
|
@ -67,13 +68,13 @@ require (
|
|||
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect
|
||||
github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d // indirect
|
||||
github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 // indirect
|
||||
github.com/pingcap/tidb/pkg/parser v0.0.0-20240430081142-7481aa6d0b8b // indirect
|
||||
github.com/pingcap/tidb/pkg/parser v0.0.0-20240703042657-230bbc2ef5ef // indirect
|
||||
github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
|
||||
github.com/prometheus/client_golang v1.19.0 // indirect
|
||||
github.com/prometheus/client_golang v1.19.1 // indirect
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.53.0 // indirect
|
||||
github.com/prometheus/common v0.55.0 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/qri-io/jsonpointer v0.1.1 // indirect
|
||||
github.com/qri-io/jsonschema v0.2.1 // indirect
|
||||
|
|
@ -105,12 +106,12 @@ require (
|
|||
golang.org/x/sys v0.21.0 // indirect
|
||||
golang.org/x/text v0.16.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
||||
golang.org/x/tools v0.22.0 // indirect
|
||||
gonum.org/v1/gonum v0.8.2 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect
|
||||
google.golang.org/grpc v1.63.2 // indirect
|
||||
google.golang.org/protobuf v1.34.1 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
|
|
@ -121,8 +122,3 @@ replace (
|
|||
|
||||
github.com/tikv/client-go/v2 => ../
|
||||
)
|
||||
|
||||
replace (
|
||||
github.com/pingcap/tidb => github.com/you06/tidb v1.1.0-beta.0.20240624051137-006cbc9cc886
|
||||
github.com/pingcap/tidb/pkg/parser => github.com/you06/tidb/pkg/parser v0.0.0-20240624051137-006cbc9cc886
|
||||
)
|
||||
|
|
|
|||
|
|
@ -311,6 +311,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
|
|||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/ncw/directio v1.0.4/go.mod h1:CKGdcN7StAaqjT7Qack3lAXeX4pjnyc46YeqZH1yWVY=
|
||||
github.com/ncw/directio v1.0.5 h1:JSUBhdjEvVaJvOoyPAbcW0fnd0tvRXD76wEfZ1KcQz4=
|
||||
github.com/ncw/directio v1.0.5/go.mod h1:rX/pKEYkOXBGOggmcyJeJGloCkleSvphPx2eV3t6ROk=
|
||||
|
|
@ -363,6 +365,10 @@ github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d h1:y3EueKVfVykdpTyfU
|
|||
github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA=
|
||||
github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 h1:T4pXRhBflzDeAhmOQHNPRRogMYxP13V7BkYw3ZsoSfE=
|
||||
github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5/go.mod h1:rlimy0GcTvjiJqvD5mXTRr8O2eNZPBrcUgiWVYp9530=
|
||||
github.com/pingcap/tidb v1.1.0-beta.0.20240703042657-230bbc2ef5ef h1:qf0QuRpodj5Yvm22BuuBoGFaiolL7KdkEYdnQ7DUstg=
|
||||
github.com/pingcap/tidb v1.1.0-beta.0.20240703042657-230bbc2ef5ef/go.mod h1:vfEq5Kh+kAxB6ROcwTubBBk1lzZmIHMVzUwFYwQSJIA=
|
||||
github.com/pingcap/tidb/pkg/parser v0.0.0-20240703042657-230bbc2ef5ef h1:HhIocGusIROnXhcXQ9DQL/+RgJpGXGlOkI/IX25pNrA=
|
||||
github.com/pingcap/tidb/pkg/parser v0.0.0-20240703042657-230bbc2ef5ef/go.mod h1:c/4la2yfv1vBYvtIG8WCDyDinLMDIUC5+zLRHiafY+Y=
|
||||
github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 h1:fVNBE06Rjec+EIHaYAKAHa/bIt5lnu3Zh9O6kV7ZAdg=
|
||||
github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||
|
|
@ -378,15 +384,15 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:Om
|
|||
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig=
|
||||
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
|
||||
github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
|
||||
github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
|
||||
github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
|
||||
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
|
||||
github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE=
|
||||
github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U=
|
||||
github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
|
||||
github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
|
||||
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
|
||||
|
|
@ -494,10 +500,6 @@ github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q
|
|||
github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 h1:mr+7gGPUasLmH3/5Iv1zwQwiY0WgGO21Ym7Q4FVw+xs=
|
||||
github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17/go.mod h1:u9udtIEWeBkphB2isZ8V8xVIMWgcUobH+7FRMO/Ld6c=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/you06/tidb v1.1.0-beta.0.20240624051137-006cbc9cc886 h1:jw3Z7XTxtzPgqUqhMQHOlmETHvk+bxR9uG+30ZtWlPQ=
|
||||
github.com/you06/tidb v1.1.0-beta.0.20240624051137-006cbc9cc886/go.mod h1:QBEHAvrZRsmpbrYcBkphRVf8tP8OaYPl2uetSb5PP64=
|
||||
github.com/you06/tidb/pkg/parser v0.0.0-20240624051137-006cbc9cc886 h1:Eqw/mDIP4bH+g/KcYZngtA1X9zQ+mZn4CMDJKBumZI8=
|
||||
github.com/you06/tidb/pkg/parser v0.0.0-20240624051137-006cbc9cc886/go.mod h1:c/4la2yfv1vBYvtIG8WCDyDinLMDIUC5+zLRHiafY+Y=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
|
|
@ -608,8 +610,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
|||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
|
||||
golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
|
|
@ -708,8 +710,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
|
|||
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
|
||||
golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
|
@ -748,8 +750,8 @@ google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
|
|||
google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
|
||||
google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9 h1:ATnmU8nL2NfIyTSiBvJVDIDIr3qBmeW+c7z7XU21eWs=
|
||||
google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9/go.mod h1:j5uROIAAgi3YmtiETMt1LW0d/lHqQ7wwrIY4uGRXLQ4=
|
||||
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
|
||||
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
|
|
|||
|
|
@ -2154,7 +2154,12 @@ func (c *RegionCache) scanRegions(bo *retry.Backoffer, startKey, endKey []byte,
|
|||
metrics.RegionCacheCounterWithScanRegionsOK.Inc()
|
||||
|
||||
if len(regionsInfo) == 0 {
|
||||
return nil, errors.Errorf("PD returned no region, limit: %d", limit)
|
||||
backoffErr = errors.Errorf("PD returned no region, limit: %d", limit)
|
||||
continue
|
||||
}
|
||||
if regionsHaveGapInRanges([]pd.KeyRange{{StartKey: startKey, EndKey: endKey}}, regionsInfo, limit) {
|
||||
backoffErr = errors.Errorf("PD returned regions have gaps, limit: %d", limit)
|
||||
continue
|
||||
}
|
||||
return c.handleRegionInfos(bo, regionsInfo, true)
|
||||
}
|
||||
|
|
@ -2210,15 +2215,78 @@ func (c *RegionCache) batchScanRegions(bo *retry.Backoffer, keyRanges []pd.KeyRa
|
|||
|
||||
metrics.RegionCacheCounterWithBatchScanRegionsOK.Inc()
|
||||
if len(regionsInfo) == 0 {
|
||||
return nil, errors.Errorf(
|
||||
backoffErr = errors.Errorf(
|
||||
"PD returned no region, range num: %d, limit: %d",
|
||||
len(keyRanges), limit,
|
||||
)
|
||||
continue
|
||||
}
|
||||
if regionsHaveGapInRanges(keyRanges, regionsInfo, limit) {
|
||||
backoffErr = errors.Errorf(
|
||||
"PD returned regions have gaps, range num: %d, limit: %d",
|
||||
len(keyRanges), limit,
|
||||
)
|
||||
continue
|
||||
}
|
||||
return c.handleRegionInfos(bo, regionsInfo, opt.needRegionHasLeaderPeer)
|
||||
}
|
||||
}
|
||||
|
||||
// regionsHaveGapInRanges checks if the loaded regions can fully cover the key ranges.
|
||||
// If there are any gaps between the regions, it returns true, then the requests might be retried.
|
||||
// TODO: remove this function after PD client supports gap detection and handling it.
|
||||
func regionsHaveGapInRanges(ranges []pd.KeyRange, regionsInfo []*pd.Region, limit int) bool {
|
||||
if len(ranges) == 0 {
|
||||
return false
|
||||
}
|
||||
if len(regionsInfo) == 0 {
|
||||
return true
|
||||
}
|
||||
checkIdx := 0 // checked index of ranges
|
||||
checkKey := ranges[0].StartKey // checked key of ranges
|
||||
for _, r := range regionsInfo {
|
||||
if r.Meta == nil {
|
||||
return true
|
||||
}
|
||||
if bytes.Compare(r.Meta.StartKey, checkKey) > 0 {
|
||||
// there is a gap between returned region's start_key and current check key
|
||||
return true
|
||||
}
|
||||
if len(r.Meta.EndKey) == 0 {
|
||||
// the current region contains all the rest ranges.
|
||||
return false
|
||||
}
|
||||
checkKey = r.Meta.EndKey
|
||||
for len(ranges[checkIdx].EndKey) > 0 && bytes.Compare(checkKey, ranges[checkIdx].EndKey) >= 0 {
|
||||
// the end_key of returned region can cover multi ranges.
|
||||
checkIdx++
|
||||
if checkIdx == len(ranges) {
|
||||
// all ranges are covered.
|
||||
return false
|
||||
}
|
||||
}
|
||||
if bytes.Compare(checkKey, ranges[checkIdx].StartKey) < 0 {
|
||||
// if check_key < start_key, move it forward to start_key.
|
||||
checkKey = ranges[checkIdx].StartKey
|
||||
}
|
||||
}
|
||||
if limit > 0 && len(regionsInfo) == limit {
|
||||
// the regionsInfo is limited by the limit, so there may be some ranges not covered.
|
||||
// But the previous regions are continuous, so we just need to check the rest ranges.
|
||||
return false
|
||||
}
|
||||
if checkIdx < len(ranges)-1 {
|
||||
// there are still some ranges not covered.
|
||||
return true
|
||||
}
|
||||
if len(checkKey) == 0 {
|
||||
return false
|
||||
} else if len(ranges[checkIdx].EndKey) == 0 {
|
||||
return true
|
||||
}
|
||||
return bytes.Compare(checkKey, ranges[checkIdx].EndKey) < 0
|
||||
}
|
||||
|
||||
func (c *RegionCache) batchScanRegionsFallback(bo *retry.Backoffer, keyRanges []pd.KeyRange, limit int, opts ...BatchLocateKeyRangesOpt) ([]*Region, error) {
|
||||
logutil.BgLogger().Warn("batch scan regions fallback to scan regions", zap.Int("range-num", len(keyRanges)))
|
||||
res := make([]*Region, 0, len(keyRanges))
|
||||
|
|
|
|||
|
|
@ -2689,3 +2689,147 @@ func (s *testRegionCacheSuite) testBatchScanRegions() {
|
|||
s.cluster.Split(newID1, newID2, []byte("e1"), newPeers, newPeers[0])
|
||||
}, []uint64{regions[1], regions[3], regions[4], newID1, regions[5], regions[6]})
|
||||
}
|
||||
|
||||
func (s *testRegionCacheSuite) TestRangesAreCoveredCheck() {
|
||||
check := func(ranges []string, regions []string, limit int, expect bool) {
|
||||
rs := make([]pd.KeyRange, 0, len(ranges)/2)
|
||||
for i := 0; i < len(ranges); i += 2 {
|
||||
rs = append(rs, pd.KeyRange{StartKey: []byte(ranges[i]), EndKey: []byte(ranges[i+1])})
|
||||
}
|
||||
rgs := make([]*pd.Region, 0, len(regions))
|
||||
for i := 0; i < len(regions); i += 2 {
|
||||
rgs = append(rgs, &pd.Region{Meta: &metapb.Region{
|
||||
StartKey: []byte(regions[i]),
|
||||
EndKey: []byte(regions[i+1]),
|
||||
}})
|
||||
}
|
||||
s.Equal(expect, regionsHaveGapInRanges(rs, rgs, limit))
|
||||
}
|
||||
boundCases := [][]string{
|
||||
{"a", "c"},
|
||||
{"a", "b", "b", "c"},
|
||||
{"a", "a1", "a1", "b", "b", "b1", "b1", "c"},
|
||||
}
|
||||
for _, boundCase := range boundCases {
|
||||
// positive
|
||||
check(boundCase, []string{"a", "c"}, -1, false)
|
||||
check(boundCase, []string{"a", ""}, -1, false)
|
||||
check(boundCase, []string{"", "c"}, -1, false)
|
||||
// negative
|
||||
check(boundCase, []string{"a", "b"}, -1, true)
|
||||
check(boundCase, []string{"b", "c"}, -1, true)
|
||||
check(boundCase, []string{"b", ""}, -1, true)
|
||||
check(boundCase, []string{"", "b"}, -1, true)
|
||||
// positive
|
||||
check(boundCase, []string{"a", "b", "b", "c"}, -1, false)
|
||||
check(boundCase, []string{"", "b", "b", "c"}, -1, false)
|
||||
check(boundCase, []string{"a", "b", "b", ""}, -1, false)
|
||||
check(boundCase, []string{"", "b", "b", ""}, -1, false)
|
||||
// negative
|
||||
check(boundCase, []string{"a", "b", "b1", "c"}, -1, true)
|
||||
check(boundCase, []string{"", "b", "b1", "c"}, -1, true)
|
||||
check(boundCase, []string{"a", "b", "b1", ""}, -1, true)
|
||||
check(boundCase, []string{"", "b", "b1", ""}, -1, true)
|
||||
check(boundCase, []string{}, -1, true)
|
||||
}
|
||||
|
||||
nonContinuousCases := [][]string{
|
||||
{"a", "b", "c", "d"},
|
||||
{"a", "b1", "b1", "b", "c", "d"},
|
||||
{"a", "b", "c", "c1", "c1", "d"},
|
||||
{"a", "b1", "b1", "b", "c", "c1", "c1", "d"},
|
||||
}
|
||||
for _, nonContinuousCase := range nonContinuousCases {
|
||||
// positive
|
||||
check(nonContinuousCase, []string{"a", "d"}, -1, false)
|
||||
check(nonContinuousCase, []string{"", "d"}, -1, false)
|
||||
check(nonContinuousCase, []string{"a", ""}, -1, false)
|
||||
check(nonContinuousCase, []string{"", ""}, -1, false)
|
||||
// negative
|
||||
check(nonContinuousCase, []string{"a", "b"}, -1, true)
|
||||
check(nonContinuousCase, []string{"b", "c"}, -1, true)
|
||||
check(nonContinuousCase, []string{"c", "d"}, -1, true)
|
||||
check(nonContinuousCase, []string{"", "b"}, -1, true)
|
||||
check(nonContinuousCase, []string{"c", ""}, -1, true)
|
||||
}
|
||||
|
||||
unboundCases := [][]string{
|
||||
{"", ""},
|
||||
{"", "b", "b", ""},
|
||||
{"", "a1", "a1", "b", "b", "b1", "b1", ""},
|
||||
}
|
||||
for _, unboundCase := range unboundCases {
|
||||
// positive
|
||||
check(unboundCase, []string{"", ""}, -1, false)
|
||||
// negative
|
||||
check(unboundCase, []string{"a", "c"}, -1, true)
|
||||
check(unboundCase, []string{"a", ""}, -1, true)
|
||||
check(unboundCase, []string{"", "c"}, -1, true)
|
||||
// positive
|
||||
check(unboundCase, []string{"", "b", "b", ""}, -1, false)
|
||||
// negative
|
||||
check(unboundCase, []string{"", "b", "b1", ""}, -1, true)
|
||||
check(unboundCase, []string{"a", "b", "b", ""}, -1, true)
|
||||
check(unboundCase, []string{"", "b", "b", "c"}, -1, true)
|
||||
check(unboundCase, []string{}, -1, true)
|
||||
}
|
||||
|
||||
// test half bounded ranges
|
||||
check([]string{"", "b"}, []string{"", "a"}, -1, true)
|
||||
check([]string{"", "b"}, []string{"", "a"}, 1, false) // it's just limitation reached
|
||||
check([]string{"", "b"}, []string{"", "a"}, 2, true)
|
||||
check([]string{"a", ""}, []string{"b", ""}, -1, true)
|
||||
check([]string{"a", ""}, []string{"b", ""}, 1, true)
|
||||
check([]string{"a", ""}, []string{"b", "c"}, 1, true)
|
||||
check([]string{"a", ""}, []string{"a", ""}, -1, false)
|
||||
}
|
||||
|
||||
func (s *testRegionCacheSuite) TestScanRegionsWithGaps() {
|
||||
// Split at "a", "c", "e"
|
||||
// nil --- 'a' --- 'c' --- 'e' --- nil
|
||||
// <- 0 -> <- 1 -> <- 2 -> <- 3 -->
|
||||
regions := s.cluster.AllocIDs(3)
|
||||
regions = append([]uint64{s.region1}, regions...)
|
||||
|
||||
peers := [][]uint64{{s.peer1, s.peer2}}
|
||||
for i := 0; i < 3; i++ {
|
||||
peers = append(peers, s.cluster.AllocIDs(2))
|
||||
}
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
s.cluster.Split(regions[i], regions[i+1], []byte{'a' + 2*byte(i)}, peers[i+1], peers[i+1][0])
|
||||
}
|
||||
|
||||
// the last region is not reported to PD yet
|
||||
getRegionIDsWithInject := func(fn func() ([]*Region, error)) []uint64 {
|
||||
s.cache.clear()
|
||||
err := failpoint.Enable("tikvclient/mockSplitRegionNotReportToPD", fmt.Sprintf(`return(%d)`, regions[2]))
|
||||
s.Nil(err)
|
||||
resCh := make(chan []*Region)
|
||||
errCh := make(chan error)
|
||||
go func() {
|
||||
rs, err := fn()
|
||||
errCh <- err
|
||||
resCh <- rs
|
||||
}()
|
||||
time.Sleep(time.Second)
|
||||
failpoint.Disable("tikvclient/mockSplitRegionNotReportToPD")
|
||||
s.Nil(<-errCh)
|
||||
rs := <-resCh
|
||||
regionIDs := make([]uint64, 0, len(rs))
|
||||
for _, r := range rs {
|
||||
regionIDs = append(regionIDs, r.GetID())
|
||||
}
|
||||
return regionIDs
|
||||
}
|
||||
|
||||
scanRegionRes := getRegionIDsWithInject(func() ([]*Region, error) {
|
||||
return s.cache.BatchLoadRegionsWithKeyRange(s.bo, []byte(""), []byte(""), 10)
|
||||
})
|
||||
s.Equal(scanRegionRes, regions)
|
||||
|
||||
batchScanRegionRes := getRegionIDsWithInject(func() ([]*Region, error) {
|
||||
return s.cache.BatchLoadRegionsWithKeyRanges(s.bo, []pd.KeyRange{{StartKey: []byte{}, EndKey: []byte{}}}, 10)
|
||||
})
|
||||
s.Equal(batchScanRegionRes, regions)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ import (
|
|||
"github.com/pingcap/kvproto/pkg/kvrpcpb"
|
||||
"github.com/pingcap/kvproto/pkg/metapb"
|
||||
"github.com/tikv/client-go/v2/internal/mockstore/cluster"
|
||||
"github.com/tikv/client-go/v2/util"
|
||||
pd "github.com/tikv/pd/client"
|
||||
)
|
||||
|
||||
|
|
@ -378,6 +379,15 @@ func (c *Cluster) ScanRegions(startKey, endKey []byte, limit int, opts ...pd.Get
|
|||
regions = regions[:endPos]
|
||||
}
|
||||
}
|
||||
if rid, err := util.EvalFailpoint("mockSplitRegionNotReportToPD"); err == nil {
|
||||
notReportRegionID := uint64(rid.(int))
|
||||
for i, r := range regions {
|
||||
if r.Meta.Id == notReportRegionID {
|
||||
regions = append(regions[:i], regions[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if limit > 0 && len(regions) > limit {
|
||||
regions = regions[:limit]
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue