diff --git a/error/error.go b/error/error.go index a3b860a7..671f6693 100644 --- a/error/error.go +++ b/error/error.go @@ -89,6 +89,8 @@ var ( ErrTiKVDiskFull = errors.New("tikv disk full") // ErrRegionRecoveryInProgress is the error when region is recovering. ErrRegionRecoveryInProgress = errors.New("region is being online unsafe recovered") + // ErrRegionFlashbackInProgress is the error when a region in the flashback progress receive any other request. + ErrRegionFlashbackInProgress = errors.New("region is in the flashback progress") // ErrUnknown is the unknow error. ErrUnknown = errors.New("unknow") // ErrResultUndetermined is the error when execution result is unknown. diff --git a/go.mod b/go.mod index 5ada7112..530bc0bd 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 - github.com/pingcap/kvproto v0.0.0-20220902075447-f95ac338b331 + github.com/pingcap/kvproto v0.0.0-20220906053631-2e37953b2b43 github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 diff --git a/go.sum b/go.sum index 702e1e28..b05d8c4d 100644 --- a/go.sum +++ b/go.sum @@ -155,8 +155,8 @@ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20220902075447-f95ac338b331 h1:LLPNqaM0C0hHER4U+cl17pvVBFuSDBbVl2oJQ6f6bg4= -github.com/pingcap/kvproto v0.0.0-20220902075447-f95ac338b331/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20220906053631-2e37953b2b43 h1:5q7Ns0R7q6Uj+fpa3lDTijrcqgId4lNdGa2AG7izB5c= +github.com/pingcap/kvproto v0.0.0-20220906053631-2e37953b2b43/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM= github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/integration_tests/go.mod b/integration_tests/go.mod index c7abc255..88fc64b6 100644 --- a/integration_tests/go.mod +++ b/integration_tests/go.mod @@ -6,7 +6,7 @@ require ( github.com/ninedraft/israce v0.0.3 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20220423142525-ae43b7f4e5c3 - github.com/pingcap/kvproto v0.0.0-20220902075447-f95ac338b331 + github.com/pingcap/kvproto v0.0.0-20220906053631-2e37953b2b43 github.com/pingcap/tidb v1.1.0-beta.0.20220902042024-0482b2e83ed2 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.0 diff --git a/integration_tests/go.sum b/integration_tests/go.sum index e9dbbd6e..00f7a850 100644 --- a/integration_tests/go.sum +++ b/integration_tests/go.sum @@ -301,10 +301,8 @@ github.com/pingcap/fn v0.0.0-20200306044125-d5540d389059 h1:Pe2LbxRmbTfAoKJ65bZL github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20220818023518-a0f02b6efcee h1:s1al2ci3MEj5VnNuUCGAfeqpbCxcMeZibOyxw8ClHLE= -github.com/pingcap/kvproto v0.0.0-20220818023518-a0f02b6efcee/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= -github.com/pingcap/kvproto v0.0.0-20220902075447-f95ac338b331 h1:LLPNqaM0C0hHER4U+cl17pvVBFuSDBbVl2oJQ6f6bg4= -github.com/pingcap/kvproto v0.0.0-20220902075447-f95ac338b331/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20220906053631-2e37953b2b43 h1:5q7Ns0R7q6Uj+fpa3lDTijrcqgId4lNdGa2AG7izB5c= +github.com/pingcap/kvproto v0.0.0-20220906053631-2e37953b2b43/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/log v1.1.0 h1:ELiPxACz7vdo1qAvvaWJg1NrYFoY6gqAh/+Uo6aXdD8= diff --git a/internal/locate/region_cache_test.go b/internal/locate/region_cache_test.go index 09142881..342e07df 100644 --- a/internal/locate/region_cache_test.go +++ b/internal/locate/region_cache_test.go @@ -1592,3 +1592,16 @@ func (s *testRegionCacheSuite) TestRemoveIntersectingRegions() { s.Equal(loc.Region.GetID(), regions[0]) s.checkCache(1) } + +func (s *testRegionCacheSuite) TestShouldNotRetryFlashbackInProgress() { + loc, err := s.cache.LocateKey(s.bo, []byte("a")) + s.NotNil(loc) + s.NoError(err) + ctx, err := s.cache.GetTiKVRPCContext(retry.NewBackofferWithVars(context.Background(), 100, nil), loc.Region, kv.ReplicaReadLeader, 0) + s.NotNil(ctx) + s.NoError(err) + reqSend := NewRegionRequestSender(s.cache, nil) + shouldRetry, err := reqSend.onRegionError(s.bo, ctx, nil, &errorpb.Error{FlashbackInProgress: &errorpb.FlashbackInProgress{}}) + s.Error(err) + s.False(shouldRetry) +} diff --git a/internal/locate/region_request.go b/internal/locate/region_request.go index 18b88ed2..d4470b0b 100644 --- a/internal/locate/region_request.go +++ b/internal/locate/region_request.go @@ -1389,6 +1389,8 @@ func regionErrorToLabel(e *errorpb.Error) string { return "disk_full" } else if e.GetRecoveryInProgress() != nil { return "recovery_in_progress" + } else if e.GetFlashbackInProgress() != nil { + return "flashback_in_progress" } return "unknown" } @@ -1446,6 +1448,14 @@ func (s *RegionRequestSender) onRegionError(bo *retry.Backoffer, ctx *RPCContext return false, nil } + // Since we expect that the workload should be stopped during the flashback progress, + // if a request meets the FlashbackInProgress error, it should stop retrying immediately + // to avoid unnecessary backoff and potential unexpected data status to the user. + if regionErr.GetFlashbackInProgress() != nil { + logutil.BgLogger().Debug("tikv reports `FlashbackInProgress`", zap.Stringer("req", req), zap.Stringer("ctx", ctx)) + return false, errors.Errorf("region %d is in flashback progress", regionErr.GetFlashbackInProgress().GetRegionId()) + } + // This peer is removed from the region. Invalidate the region since it's too stale. if regionErr.GetRegionNotFound() != nil { s.regionCache.InvalidateCachedRegion(ctx.Region)