From 84e019eae523b14c6f987faafb326e352f69ebbe Mon Sep 17 00:00:00 2001 From: you06 Date: Mon, 7 Jul 2025 15:38:42 +0900 Subject: [PATCH] region cache: do not invalidate the valid region cache when new region cache is loaded (#1698) Signed-off-by: you06 --- examples/gcworker/go.mod | 4 +- examples/rawkv/go.mod | 4 +- examples/txnkv/1pc_txn/go.mod | 4 +- examples/txnkv/async_commit/go.mod | 4 +- examples/txnkv/delete_range/go.mod | 4 +- examples/txnkv/go.mod | 4 +- examples/txnkv/pessimistic_txn/go.mod | 4 +- examples/txnkv/unsafedestoryrange/go.mod | 4 +- internal/locate/region_cache.go | 4 +- internal/locate/region_cache_test.go | 90 ++++++++++++++++++++++++ 10 files changed, 108 insertions(+), 18 deletions(-) diff --git a/examples/gcworker/go.mod b/examples/gcworker/go.mod index 6d2132b5..8774d94b 100644 --- a/examples/gcworker/go.mod +++ b/examples/gcworker/go.mod @@ -23,7 +23,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122 // indirect github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect - github.com/pingcap/kvproto v0.0.0-20250605100108-dc99a8f6e348 // indirect + github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3 // indirect github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.20.5 // indirect @@ -32,7 +32,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect - github.com/tikv/pd/client v0.0.0-20250528071146-6cd8a77df2fa // indirect + github.com/tikv/pd/client v0.0.0-20250625073039-fb496b371ff3 // indirect github.com/twmb/murmur3 v1.1.3 // indirect go.etcd.io/etcd/api/v3 v3.5.10 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect diff --git a/examples/rawkv/go.mod b/examples/rawkv/go.mod index 3c8aa670..66a9cc27 100644 --- a/examples/rawkv/go.mod +++ b/examples/rawkv/go.mod @@ -23,7 +23,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122 // indirect github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect - github.com/pingcap/kvproto v0.0.0-20250605100108-dc99a8f6e348 // indirect + github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3 // indirect github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.20.5 // indirect @@ -32,7 +32,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect - github.com/tikv/pd/client v0.0.0-20250528071146-6cd8a77df2fa // indirect + github.com/tikv/pd/client v0.0.0-20250625073039-fb496b371ff3 // indirect github.com/twmb/murmur3 v1.1.3 // indirect go.etcd.io/etcd/api/v3 v3.5.10 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect diff --git a/examples/txnkv/1pc_txn/go.mod b/examples/txnkv/1pc_txn/go.mod index 79ddfa1e..8f155970 100644 --- a/examples/txnkv/1pc_txn/go.mod +++ b/examples/txnkv/1pc_txn/go.mod @@ -23,7 +23,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122 // indirect github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect - github.com/pingcap/kvproto v0.0.0-20250605100108-dc99a8f6e348 // indirect + github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3 // indirect github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.20.5 // indirect @@ -32,7 +32,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect - github.com/tikv/pd/client v0.0.0-20250528071146-6cd8a77df2fa // indirect + github.com/tikv/pd/client v0.0.0-20250625073039-fb496b371ff3 // indirect github.com/twmb/murmur3 v1.1.3 // indirect go.etcd.io/etcd/api/v3 v3.5.10 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect diff --git a/examples/txnkv/async_commit/go.mod b/examples/txnkv/async_commit/go.mod index c6b354d4..93c3d661 100644 --- a/examples/txnkv/async_commit/go.mod +++ b/examples/txnkv/async_commit/go.mod @@ -23,7 +23,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122 // indirect github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect - github.com/pingcap/kvproto v0.0.0-20250605100108-dc99a8f6e348 // indirect + github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3 // indirect github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.20.5 // indirect @@ -32,7 +32,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect - github.com/tikv/pd/client v0.0.0-20250528071146-6cd8a77df2fa // indirect + github.com/tikv/pd/client v0.0.0-20250625073039-fb496b371ff3 // indirect github.com/twmb/murmur3 v1.1.3 // indirect go.etcd.io/etcd/api/v3 v3.5.10 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect diff --git a/examples/txnkv/delete_range/go.mod b/examples/txnkv/delete_range/go.mod index 6eca410e..b2b36cf1 100644 --- a/examples/txnkv/delete_range/go.mod +++ b/examples/txnkv/delete_range/go.mod @@ -23,7 +23,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122 // indirect github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect - github.com/pingcap/kvproto v0.0.0-20250605100108-dc99a8f6e348 // indirect + github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3 // indirect github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.20.5 // indirect @@ -32,7 +32,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect - github.com/tikv/pd/client v0.0.0-20250528071146-6cd8a77df2fa // indirect + github.com/tikv/pd/client v0.0.0-20250625073039-fb496b371ff3 // indirect github.com/twmb/murmur3 v1.1.3 // indirect go.etcd.io/etcd/api/v3 v3.5.10 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect diff --git a/examples/txnkv/go.mod b/examples/txnkv/go.mod index 6ac58137..753cee1e 100644 --- a/examples/txnkv/go.mod +++ b/examples/txnkv/go.mod @@ -23,7 +23,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122 // indirect github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect - github.com/pingcap/kvproto v0.0.0-20250605100108-dc99a8f6e348 // indirect + github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3 // indirect github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.20.5 // indirect @@ -32,7 +32,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect - github.com/tikv/pd/client v0.0.0-20250528071146-6cd8a77df2fa // indirect + github.com/tikv/pd/client v0.0.0-20250625073039-fb496b371ff3 // indirect github.com/twmb/murmur3 v1.1.3 // indirect go.etcd.io/etcd/api/v3 v3.5.10 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect diff --git a/examples/txnkv/pessimistic_txn/go.mod b/examples/txnkv/pessimistic_txn/go.mod index 67bb9bbe..7d9626aa 100644 --- a/examples/txnkv/pessimistic_txn/go.mod +++ b/examples/txnkv/pessimistic_txn/go.mod @@ -23,7 +23,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122 // indirect github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect - github.com/pingcap/kvproto v0.0.0-20250605100108-dc99a8f6e348 // indirect + github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3 // indirect github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.20.5 // indirect @@ -32,7 +32,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect - github.com/tikv/pd/client v0.0.0-20250528071146-6cd8a77df2fa // indirect + github.com/tikv/pd/client v0.0.0-20250625073039-fb496b371ff3 // indirect github.com/twmb/murmur3 v1.1.3 // indirect go.etcd.io/etcd/api/v3 v3.5.10 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect diff --git a/examples/txnkv/unsafedestoryrange/go.mod b/examples/txnkv/unsafedestoryrange/go.mod index cc67adc7..fa3e5860 100644 --- a/examples/txnkv/unsafedestoryrange/go.mod +++ b/examples/txnkv/unsafedestoryrange/go.mod @@ -23,7 +23,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pingcap/errors v0.11.5-0.20241219054535-6b8c588c3122 // indirect github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect - github.com/pingcap/kvproto v0.0.0-20250605100108-dc99a8f6e348 // indirect + github.com/pingcap/kvproto v0.0.0-20250616075548-d951fb623bb3 // indirect github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.20.5 // indirect @@ -32,7 +32,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect - github.com/tikv/pd/client v0.0.0-20250528071146-6cd8a77df2fa // indirect + github.com/tikv/pd/client v0.0.0-20250625073039-fb496b371ff3 // indirect github.com/twmb/murmur3 v1.1.3 // indirect go.etcd.io/etcd/api/v3 v3.5.10 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect diff --git a/internal/locate/region_cache.go b/internal/locate/region_cache.go index 22d16154..4fa8d3fe 100644 --- a/internal/locate/region_cache.go +++ b/internal/locate/region_cache.go @@ -1817,7 +1817,7 @@ func (c *RegionCache) BatchLoadRegionsWithKeyRange(bo *retry.Backoffer, startKey // TODO(youjiali1995): scanRegions always fetch regions from PD and these regions don't contain buckets information // for less traffic, so newly inserted regions in region cache don't have buckets information. We should improve it. for _, region := range regions { - c.insertRegionToCache(region, true, false) + c.insertRegionToCache(region, false, false) } return @@ -1842,7 +1842,7 @@ func (c *RegionCache) BatchLoadRegionsWithKeyRanges(bo *retry.Backoffer, keyRang defer c.mu.Unlock() for _, region := range regions { - c.insertRegionToCache(region, true, false) + c.insertRegionToCache(region, false, false) } return } diff --git a/internal/locate/region_cache_test.go b/internal/locate/region_cache_test.go index 0cb8ca5e..10dbac9d 100644 --- a/internal/locate/region_cache_test.go +++ b/internal/locate/region_cache_test.go @@ -2996,3 +2996,93 @@ func BenchmarkBatchLocateKeyRangesFromCache(t *testing.B) { } s.TearDownTest() } + +func (s *testRegionCacheSuite) TestRegionCacheValidAfterLoading() { + s.cache.clear() + + // Split regions at "a", "b", "c", ..., "j" + regions := make([]uint64, 0, 11) + region1 := s.region1 + regions = append(regions, region1) + for i := 0; i < 10; i++ { + region2 := s.cluster.AllocID() + newPeers := s.cluster.AllocIDs(2) + s.cluster.Split(region1, region2, []byte{'a' + byte(i)}, newPeers, newPeers[0]) + region1 = region2 + regions = append(regions, region1) + } + + fns := []func(){ + func() { + _, err := s.cache.LocateKeyRange(s.bo, []byte("a"), []byte("e")) + s.Nil(err) + }, + func() { + _, err := s.cache.BatchLocateKeyRanges(s.bo, []kv.KeyRange{{StartKey: []byte("a"), EndKey: []byte("e")}}) + s.Nil(err) + }, + func() { + _, err := s.cache.LocateKey(s.bo, []byte("b")) + s.Nil(err) + }, + func() { + _, err := s.cache.LocateEndKey(s.bo, []byte("c")) + s.Nil(err) + }, + func() { + for _, regionID := range regions { + _, err := s.cache.LocateRegionByID(s.bo, regionID) + s.Nil(err) + } + }, + func() { + _, _, err := s.cache.GroupKeysByRegion(s.bo, [][]byte{[]byte("a"), []byte("b"), []byte("c")}, nil) + s.Nil(err) + }, + func() { + _, err := s.cache.ListRegionIDsInKeyRange(s.bo, []byte("a"), []byte("e")) + s.Nil(err) + }, + func() { + _, err := s.cache.LoadRegionsInKeyRange(s.bo, []byte("a"), []byte("e")) + s.Nil(err) + }, + func() { + _, err := s.cache.BatchLoadRegionsWithKeyRange(s.bo, []byte("a"), []byte("e"), 10) + s.Nil(err) + }, + func() { + _, err := s.cache.BatchLoadRegionsWithKeyRanges(s.bo, []router.KeyRange{{StartKey: []byte("a"), EndKey: []byte("e")}}, 10) + s.Nil(err) + }, + func() { + _, err := s.cache.BatchLoadRegionsFromKey(s.bo, []byte("a"), 10) + s.Nil(err) + }, + } + + // Whether the region is loaded from PD(bypass the region cache) or from the region cache, + // the existing valid region should not be invalidated. + for _, fn := range fns { + loc, err := s.cache.LocateKey(s.bo, []byte("b")) + s.Nil(err) + region := s.cache.GetCachedRegionWithRLock(loc.Region) + fn() + s.True(region.isValid()) + } + + // If the region is invalidated already, it should be reloaded from PD and inserted into region cache anyway. + for _, fn := range fns { + loc, err := s.cache.LocateKey(s.bo, []byte("b")) + s.Nil(err) + region := s.cache.GetCachedRegionWithRLock(loc.Region) + region.invalidate(Other) + s.False(region.isValid()) + fn() + s.False(region.isValid()) + newLoc := s.cache.TryLocateKey([]byte("b")) + s.NotNil(newLoc) + region = s.cache.GetCachedRegionWithRLock(newLoc.Region) + s.True(region.isValid()) + } +}