Skip to content

Commit cc6166f

Browse files
MyonKemintati-chi-bot
MyonKeminta
authored and
ti-chi-bot
committed
This is an automated cherry-pick of tikv#9777
ref tikv#8978 Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io>
1 parent 54510bd commit cc6166f

File tree

6 files changed

+754
-99
lines changed

6 files changed

+754
-99
lines changed

‎pkg/gc/gc_state_manager.go‎

Lines changed: 55 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -664,39 +664,66 @@ func (m *GCStateManager) GetGCState(keyspaceID uint32) (GCState, error) {
664664
// must be fetched AFTER the beginning of the current invocation, and it never reuses the result of invocations that
665665
// started earlier than the current one.
666666
func (m *GCStateManager) GetAllKeyspacesGCStates(ctx context.Context) (map[uint32]GCState, error) {
667-
return m.allKeyspacesGCStatesSingleFlight.Do(ctx, func() (map[uint32]GCState, error) {
668-
result, err := m.getAllKeyspacesGCStatesImpl()
667+
return m.allKeyspacesGCStatesSingleFlight.Do(ctx, func(execCtx context.Context) (map[uint32]GCState, error) {
668+
result, err := m.getAllKeyspacesGCStatesImpl(execCtx)
669669
failpoint.Inject("onGetAllKeyspacesGCStatesFinish", func() {})
670670
return result, err
671671
})
672672
}
673673

674-
func (m *GCStateManager) getAllKeyspacesGCStatesImpl() (map[uint32]GCState, error) {
674+
func (m *GCStateManager) getAllKeyspacesGCStatesImpl(ctx context.Context) (map[uint32]GCState, error) {
675675
failpoint.InjectCall("onGetAllKeyspacesGCStatesStart")
676676

677-
// TODO: Handle the case that there are too many keyspaces and loading them at once is not suitable.
678-
allKeyspaces, err := m.keyspaceManager.LoadRangeKeyspace(0, 0)
679-
if err != nil {
680-
return nil, err
677+
mutexLocked := false
678+
lock := func() {
679+
m.mu.Lock()
680+
mutexLocked = true
681+
}
682+
unlock := func() {
683+
m.mu.Unlock()
684+
mutexLocked = false
681685
}
682-
m.mu.Lock()
683-
defer m.mu.Unlock()
686+
687+
ensureUnlocked := func() {
688+
if mutexLocked {
689+
unlock()
690+
}
691+
}
692+
defer ensureUnlocked()
693+
694+
keyspaceIterator := m.keyspaceManager.IterateKeyspaces()
684695

685696
// Do not guarantee atomicity among different keyspaces here.
686697
results := make(map[uint32]GCState)
687-
err = m.gcMetaStorage.RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error {
698+
lock()
699+
err := m.gcMetaStorage.RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error {
688700
nullKeyspaceState, err1 := m.getGCStateInTransaction(constant.NullKeyspaceID, wb)
689701
if err1 != nil {
690702
return err1
691703
}
692704
results[constant.NullKeyspaceID] = nullKeyspaceState
693705
return nil
694706
})
707+
unlock()
695708
if err != nil {
696709
return nil, err
697710
}
698711

699-
for _, keyspaceMeta := range allKeyspaces {
712+
for {
713+
select {
714+
case <-ctx.Done():
715+
return nil, ctx.Err()
716+
default:
717+
}
718+
719+
keyspaceMeta, ok, err := keyspaceIterator.Next()
720+
if err != nil {
721+
return nil, err
722+
}
723+
if !ok {
724+
break
725+
}
726+
700727
// Just handle the active keyspace, leave the others up to keyspace management.
701728
if keyspaceMeta.State != keyspacepb.KeyspaceState_ENABLED {
702729
continue
@@ -710,6 +737,7 @@ func (m *GCStateManager) getAllKeyspacesGCStatesImpl() (map[uint32]GCState, erro
710737
continue
711738
}
712739

740+
lock()
713741
err = m.gcMetaStorage.RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error {
714742
state, err1 := m.getGCStateInTransaction(keyspaceMeta.Id, wb)
715743
if err1 != nil {
@@ -718,6 +746,7 @@ func (m *GCStateManager) getAllKeyspacesGCStatesImpl() (map[uint32]GCState, erro
718746
results[keyspaceMeta.Id] = state
719747
return nil
720748
})
749+
unlock()
721750
if err != nil {
722751
return nil, err
723752
}
@@ -802,7 +831,7 @@ func (m *GCStateManager) CompatibleUpdateServiceGCSafePoint(keyspaceID uint32, s
802831
var txnSafePoint uint64
803832
err := m.gcMetaStorage.RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error {
804833
var err1 error
805-
txnSafePoint, _, _, err1 = m.getAllKeyspacesMaxTxnSafePoint(wb)
834+
txnSafePoint, _, _, err1 = m.getMaxTxnSafePointAmongAllKeyspaces(wb)
806835
return err1
807836
})
808837
if err != nil {
@@ -902,16 +931,20 @@ func (m *GCStateManager) SetGlobalGCBarrier(ctx context.Context, barrierID strin
902931
return m.setGlobalGCBarrierImpl(ctx, barrierID, barrierTS, ttl, now)
903932
}
904933

905-
// getAllKeyspacesMaxTxnSafePoint must be called inside a transaction,
934+
// getMaxTxnSafePointAmongAllKeyspaces must be called inside a transaction,
906935
// The WriteBatch parameter in function signature is deliberate to the call safe, do not pass nil.
907-
func (m *GCStateManager) getAllKeyspacesMaxTxnSafePoint(_ *endpoint.GCStateWriteBatch) (maxTxnSafePoint uint64, keyspaceName string, keyspaceID uint32, err error) {
908-
// TODO: Handle the case that there are too many keyspaces and loading them at once is not suitable.
909-
allKeyspaces, err1 := m.keyspaceManager.LoadRangeKeyspace(0, 0)
910-
if err1 != nil {
911-
err = err1
912-
return
913-
}
914-
for _, keyspaceMeta := range allKeyspaces {
936+
func (m *GCStateManager) getMaxTxnSafePointAmongAllKeyspaces(_ *endpoint.GCStateWriteBatch) (maxTxnSafePoint uint64, keyspaceName string, keyspaceID uint32, err error) {
937+
keyspaceIterator := m.keyspaceManager.IterateKeyspaces()
938+
for {
939+
keyspaceMeta, ok, err2 := keyspaceIterator.Next()
940+
if err2 != nil {
941+
err = err2
942+
return
943+
}
944+
if !ok {
945+
break
946+
}
947+
915948
if keyspaceMeta.State != keyspacepb.KeyspaceState_ENABLED {
916949
continue
917950
}
@@ -949,7 +982,7 @@ func (m *GCStateManager) setGlobalGCBarrierImpl(_ context.Context, barrierID str
949982
newBarrier := endpoint.NewGlobalGCBarrier(barrierID, barrierTS, expirationTime)
950983
err := m.gcMetaStorage.RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error {
951984
// Make sure global barrier ts is ahead of txn safe point of all keyspaces.
952-
maxTxnSafePoint, keyspaceName, keyspaceID, err := m.getAllKeyspacesMaxTxnSafePoint(wb)
985+
maxTxnSafePoint, keyspaceName, keyspaceID, err := m.getMaxTxnSafePointAmongAllKeyspaces(wb)
953986
if err != nil {
954987
return err
955988
}

‎pkg/gc/gc_state_manager_test.go‎

Lines changed: 104 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,20 @@ type newGCStateManagerForTestOptions struct {
9494
etcdServerCfgModifier func(cfg *embed.Config)
9595
}
9696

97+
func (opt *newGCStateManagerForTestOptions) generateKeyspacesByCount(count int) {
98+
createTime := time.Now().Unix()
99+
for i := range count {
100+
id := new(uint32)
101+
*id = uint32(i + 1)
102+
opt.specifyInitialKeyspaces = append(opt.specifyInitialKeyspaces, &keyspace.CreateKeyspaceByIDRequest{
103+
ID: id,
104+
Name: fmt.Sprintf("ks%d", *id),
105+
Config: map[string]string{keyspace.GCManagementType: keyspace.KeyspaceLevelGC},
106+
CreateTime: createTime,
107+
})
108+
}
109+
}
110+
97111
func newGCStateManagerForTest(t testing.TB, opt newGCStateManagerForTestOptions) (storage *endpoint.StorageEndpoint, provider endpoint.GCStateProvider, gcStateManager *GCStateManager, clean func(), cancel context.CancelFunc) {
98112
cfg := config.NewConfig()
99113
re := require.New(t)
@@ -127,6 +141,7 @@ func newGCStateManagerForTest(t testing.TB, opt newGCStateManagerForTestOptions)
127141

128142
// The bootstrap keyspace (DefaultKeyspaceID or SystemKeyspaceID) exists automatically after bootstrapping.
129143
if opt.specifyInitialKeyspaces == nil {
144+
<<<<<<< HEAD
130145
// In NextGen, all keyspaces should use keyspace_level GC
131146
// In Classic, we can have different GC management types
132147
var ks1Config map[string]string
@@ -135,6 +150,8 @@ func newGCStateManagerForTest(t testing.TB, opt newGCStateManagerForTestOptions)
135150
} else {
136151
ks1Config = map[string]string{"gc_management_type": "unified"}
137152
}
153+
=======
154+
>>>>>>> 7cbc3821d (gc: Optimize GetAllKeyspacesGCStates for scenarios where there are too many keyspaces (#9777))
138155
id := new(uint32)
139156
*id = 1
140157
ks1, err := keyspaceManager.CreateKeyspaceByID(&keyspace.CreateKeyspaceByIDRequest{
@@ -1868,7 +1885,7 @@ func (s *gcStateManagerTestSuite) TestGetAllKeyspacesMaxTxnSafePoint() {
18681885
var keyspaceID uint32
18691886
err := s.provider.RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error {
18701887
var err1 error
1871-
txnSafePoint, keyspaceName, keyspaceID, err1 = s.manager.getAllKeyspacesMaxTxnSafePoint(wb)
1888+
txnSafePoint, keyspaceName, keyspaceID, err1 = s.manager.getMaxTxnSafePointAmongAllKeyspaces(wb)
18721889
return err1
18731890
})
18741891
re.NoError(err)
@@ -1883,7 +1900,7 @@ func (s *gcStateManagerTestSuite) TestGetAllKeyspacesMaxTxnSafePoint() {
18831900
}
18841901
err = s.provider.RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error {
18851902
var err1 error
1886-
txnSafePoint, keyspaceName, keyspaceID, err1 = s.manager.getAllKeyspacesMaxTxnSafePoint(wb)
1903+
txnSafePoint, keyspaceName, keyspaceID, err1 = s.manager.getMaxTxnSafePointAmongAllKeyspaces(wb)
18871904
return err1
18881905
})
18891906
re.NoError(err)
@@ -2067,6 +2084,88 @@ func (s *gcStateManagerTestSuite) TestGetAllKeyspacesGCStatesConcurrentCallShari
20672084
re.Equal(int64(2), executionCount.Load())
20682085
}
20692086

2087+
func TestGetAllKeysapcesGCStatesOnTooManyKeyspaces(t *testing.T) {
2088+
re := require.New(t)
2089+
2090+
const totalKeyspaces = keyspace.IteratorLoadingBatchSize * 3
2091+
2092+
opt := newGCStateManagerForTestOptions{
2093+
specifyInitialKeyspaces: make([]*keyspace.CreateKeyspaceByIDRequest, 0, totalKeyspaces),
2094+
}
2095+
opt.generateKeyspacesByCount(totalKeyspaces)
2096+
2097+
_, _, gcStateManager, clean, cancel := newGCStateManagerForTest(t, opt)
2098+
defer func() {
2099+
cancel()
2100+
clean()
2101+
}()
2102+
2103+
gcStates, err := gcStateManager.GetAllKeyspacesGCStates(context.Background())
2104+
re.Len(gcStates, totalKeyspaces+2) // Including the null keyspace, the default keyspace or the system keyspace.
2105+
2106+
re.NoError(err)
2107+
keyspaceIDs := make([]uint32, 0, len(gcStates))
2108+
for keyspaceID, gcState := range gcStates {
2109+
re.Equal(keyspaceID, gcState.KeyspaceID)
2110+
keyspaceIDs = append(keyspaceIDs, keyspaceID)
2111+
}
2112+
slices.Sort(keyspaceIDs)
2113+
2114+
expectedKeyspaceIDs := make([]uint32, 0, len(gcStates))
2115+
if !kerneltype.IsNextGen() {
2116+
expectedKeyspaceIDs = append(expectedKeyspaceIDs, constant.DefaultKeyspaceID)
2117+
}
2118+
for i := range totalKeyspaces {
2119+
expectedKeyspaceIDs = append(expectedKeyspaceIDs, uint32(i+1))
2120+
}
2121+
if kerneltype.IsNextGen() {
2122+
expectedKeyspaceIDs = append(expectedKeyspaceIDs, constant.SystemKeyspaceID)
2123+
}
2124+
expectedKeyspaceIDs = append(expectedKeyspaceIDs, constant.NullKeyspaceID)
2125+
re.Equal(expectedKeyspaceIDs, keyspaceIDs)
2126+
}
2127+
2128+
func TestGetMaxTxnSafePointAmongAllKeyspacesOnTooManyKeyspaces(t *testing.T) {
2129+
re := require.New(t)
2130+
2131+
const totalKeyspaces = keyspace.IteratorLoadingBatchSize * 2
2132+
2133+
opt := newGCStateManagerForTestOptions{
2134+
specifyInitialKeyspaces: make([]*keyspace.CreateKeyspaceByIDRequest, 0, totalKeyspaces),
2135+
}
2136+
opt.generateKeyspacesByCount(totalKeyspaces)
2137+
2138+
_, _, gcStateManager, clean, cancel := newGCStateManagerForTest(t, opt)
2139+
defer func() {
2140+
cancel()
2141+
clean()
2142+
}()
2143+
2144+
now := time.Now()
2145+
// Test around the boundary of two loading batches, so that it's likely to detect incorrectness when loading
2146+
// multiple batches.
2147+
for i := keyspace.IteratorLoadingBatchSize - 5; i <= keyspace.IteratorLoadingBatchSize+5; i++ {
2148+
keyspaceID := uint32(i)
2149+
newTxnSafePoint := uint64(i)
2150+
res, err := gcStateManager.AdvanceTxnSafePoint(keyspaceID, newTxnSafePoint, now)
2151+
re.NoError(err)
2152+
re.Equal(newTxnSafePoint, res.NewTxnSafePoint)
2153+
2154+
var maxTxnSafePoint uint64
2155+
var keyspaceIDWithMaxTxnSafePoint uint32
2156+
var keyspaceNameWithMaxTxnSafePoint string
2157+
err = gcStateManager.gcMetaStorage.RunInGCStateTransaction(func(wb *endpoint.GCStateWriteBatch) error {
2158+
var err1 error
2159+
maxTxnSafePoint, keyspaceNameWithMaxTxnSafePoint, keyspaceIDWithMaxTxnSafePoint, err1 = gcStateManager.getMaxTxnSafePointAmongAllKeyspaces(wb)
2160+
return err1
2161+
})
2162+
re.NoError(err)
2163+
re.Equal(newTxnSafePoint, maxTxnSafePoint)
2164+
re.Equal(keyspaceID, keyspaceIDWithMaxTxnSafePoint)
2165+
re.Equal(fmt.Sprintf("ks%d", keyspaceID), keyspaceNameWithMaxTxnSafePoint)
2166+
}
2167+
}
2168+
20702169
func benchmarkGetAllKeyspacesGCStatesImpl(b *testing.B, keyspacesCount int, parallelism int) {
20712170
re := require.New(b)
20722171
fname := testutil.InitTempFileLogger("info")
@@ -2078,7 +2177,10 @@ func benchmarkGetAllKeyspacesGCStatesImpl(b *testing.B, keyspacesCount int, para
20782177
cfg.LogOutputs = []string{fname}
20792178
},
20802179
}
2180+
<<<<<<< HEAD
20812181

2182+
=======
2183+
>>>>>>> 7cbc3821d (gc: Optimize GetAllKeyspacesGCStates for scenarios where there are too many keyspaces (#9777))
20822184
createTime := time.Now().Unix()
20832185
for i := range keyspacesCount {
20842186
id := new(uint32)

0 commit comments

Comments
 (0)
⚡