Skip to content

Commit 2baf2a5

Browse files
AmoebaProtozoaCopilot
AmoebaProtozoa
and
Copilot
authored
keyspace: cache meta-service group status (tikv#463)
* cache meta-service group status Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * fix lint Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * fix lint Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * fix deadlock Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * cleanup Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * assign to group no longer error Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * lint Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * fix integration test Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * leader callback Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * add more tests Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * lint Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * don't error keyspace udpate when assignment count failed Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * add some comments Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * don't flush on follower Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * add comments for leader check Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * improve comments Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> --------- Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent 76277a6 commit 2baf2a5

File tree

6 files changed

+451
-185
lines changed

6 files changed

+451
-185
lines changed

‎pkg/keyspace/keyspace.go‎

Lines changed: 49 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -322,15 +322,7 @@ func (manager *Manager) CreateKeyspace(request *CreateKeyspaceRequest) (*keyspac
322322
len(manager.mgm.GetGroups()) > 0
323323
if assignToMetaServiceGroup {
324324
stepStart = time.Now()
325-
metaServiceGroup, err := manager.mgm.AssignToGroup(1)
326-
if err != nil {
327-
log.Error("[create-keyspace] failed to assign keyspace to meta-service group",
328-
zap.Uint32("keyspace-id", newID),
329-
zap.String("keyspace-name", request.Name),
330-
zap.Error(err),
331-
)
332-
return nil, err
333-
}
325+
metaServiceGroup := manager.mgm.AssignToGroup(1)
334326
if metaServiceGroup != "" {
335327
request.Config[MetaServiceGroupIDKey] = metaServiceGroup
336328
}
@@ -611,6 +603,11 @@ const (
611603
func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation) (*keyspacepb.KeyspaceMeta, error) {
612604
var meta *keyspacepb.KeyspaceMeta
613605
oldConfig := make(map[string]string)
606+
var (
607+
shouldUpdateMetaServiceGroup bool
608+
oldMetaServiceGroup string
609+
newMetaServiceGroup string
610+
)
614611
err := manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error {
615612
// First get KeyspaceID from ID.
616613
loaded, id, err := manager.store.LoadKeyspaceID(txn, name)
@@ -664,12 +661,13 @@ func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation)
664661
}
665662
}
666663
// If the assigned meta-service group changed, need to update the meta-service group assignment count.
667-
oldMetaServiceGroup := oldConfig[MetaServiceGroupIDKey]
668-
newMetaServiceGroup := newConfig[MetaServiceGroupIDKey]
664+
oldMetaServiceGroup = oldConfig[MetaServiceGroupIDKey]
665+
newMetaServiceGroup = newConfig[MetaServiceGroupIDKey]
669666
if manager.mgm != nil && oldMetaServiceGroup != newMetaServiceGroup {
670-
if err := manager.mgm.UpdateAssignment(oldMetaServiceGroup, newMetaServiceGroup); err != nil {
671-
return err
667+
if newMetaServiceGroup != "" && !manager.mgm.HasGroup(newMetaServiceGroup) {
668+
return errUnknownMetaServiceGroup
672669
}
670+
shouldUpdateMetaServiceGroup = true
673671
}
674672
// Save the updated keyspace meta.
675673
if err := manager.store.SaveKeyspaceMeta(txn, meta); err != nil {
@@ -690,6 +688,15 @@ func (manager *Manager) UpdateKeyspaceConfig(name string, mutations []*Mutation)
690688
)
691689
return nil, err
692690
}
691+
if shouldUpdateMetaServiceGroup && manager.mgm != nil {
692+
if err := manager.mgm.UpdateAssignment(oldMetaServiceGroup, newMetaServiceGroup); err != nil {
693+
log.Warn("[keyspace] failed to update meta-service group assignment",
694+
zap.String("old-group", oldMetaServiceGroup),
695+
zap.String("new-group", newMetaServiceGroup),
696+
zap.Error(err),
697+
)
698+
}
699+
}
693700
if manager.mgm != nil {
694701
manager.mgm.AttachEndpoints(meta.GetConfig())
695702
}
@@ -712,6 +719,7 @@ func (manager *Manager) UpdateKeyspaceStateByName(name string, newState keyspace
712719
return nil, ErrModifyDefaultKeyspace
713720
}
714721
var meta *keyspacepb.KeyspaceMeta
722+
var removedMetaServiceGroup string
715723
err := manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error {
716724
// First get KeyspaceID from ID.
717725
loaded, id, err := manager.store.LoadKeyspaceID(txn, name)
@@ -732,7 +740,8 @@ func (manager *Manager) UpdateKeyspaceStateByName(name string, newState keyspace
732740
return ErrKeyspaceNotFound
733741
}
734742
// Update keyspace meta.
735-
if err = manager.updateKeyspaceState(txn, meta, newState, now); err != nil {
743+
removedMetaServiceGroup, err = manager.updateKeyspaceState(meta, newState, now)
744+
if err != nil {
736745
return err
737746
}
738747
return manager.store.SaveKeyspaceMeta(txn, meta)
@@ -745,6 +754,14 @@ func (manager *Manager) UpdateKeyspaceStateByName(name string, newState keyspace
745754
)
746755
return nil, err
747756
}
757+
if manager.mgm != nil && removedMetaServiceGroup != "" {
758+
if err := manager.mgm.UpdateAssignment(removedMetaServiceGroup, ""); err != nil {
759+
log.Warn("[keyspace] failed to update meta-service group assignment",
760+
zap.String("old-group", removedMetaServiceGroup),
761+
zap.Error(err),
762+
)
763+
}
764+
}
748765
log.Info("[keyspace] keyspace state updated",
749766
zap.Uint32("ID", meta.GetId()),
750767
zap.String("keyspace-id", meta.GetName()),
@@ -753,23 +770,6 @@ func (manager *Manager) UpdateKeyspaceStateByName(name string, newState keyspace
753770
return meta, nil
754771
}
755772

756-
func (manager *Manager) removeKeyspaceFromMSGroup(
757-
txn kv.Txn,
758-
keyspaceMeta *keyspacepb.KeyspaceMeta,
759-
oldMetaServiceGroup string,
760-
removeMetaServiceGroup string,
761-
) error {
762-
// Remove keyspace config meta-service-group-id.
763-
delete(keyspaceMeta.Config, MetaServiceGroupIDKey)
764-
// Remove keyspace from meta-service group.
765-
if manager.mgm != nil && oldMetaServiceGroup != removeMetaServiceGroup {
766-
if err := manager.mgm.UpdateAssignmentWithTxn(txn, oldMetaServiceGroup, removeMetaServiceGroup); err != nil {
767-
return err
768-
}
769-
}
770-
return nil
771-
}
772-
773773
// UpdateKeyspaceStateByID updates target keyspace to the given state if it's not already in that state.
774774
// It returns error if saving failed, operation not allowed, or if keyspace not exists.
775775
func (manager *Manager) UpdateKeyspaceStateByID(id uint32, newState keyspacepb.KeyspaceState, now int64) (*keyspacepb.KeyspaceMeta, error) {
@@ -782,6 +782,7 @@ func (manager *Manager) UpdateKeyspaceStateByID(id uint32, newState keyspacepb.K
782782
}
783783
var meta *keyspacepb.KeyspaceMeta
784784
var err error
785+
var removedMetaServiceGroup string
785786
err = manager.store.RunInTxn(manager.ctx, func(txn kv.Txn) error {
786787
manager.metaLock.Lock(id)
787788
defer manager.metaLock.Unlock(id)
@@ -794,7 +795,8 @@ func (manager *Manager) UpdateKeyspaceStateByID(id uint32, newState keyspacepb.K
794795
return ErrKeyspaceNotFound
795796
}
796797
// Update keyspace meta.
797-
if err = manager.updateKeyspaceState(txn, meta, newState, now); err != nil {
798+
removedMetaServiceGroup, err = manager.updateKeyspaceState(meta, newState, now)
799+
if err != nil {
798800
return err
799801
}
800802
return manager.store.SaveKeyspaceMeta(txn, meta)
@@ -807,6 +809,14 @@ func (manager *Manager) UpdateKeyspaceStateByID(id uint32, newState keyspacepb.K
807809
)
808810
return nil, err
809811
}
812+
if manager.mgm != nil && removedMetaServiceGroup != "" {
813+
if err := manager.mgm.UpdateAssignment(removedMetaServiceGroup, ""); err != nil {
814+
log.Warn("[keyspace] failed to update meta-service group assignment",
815+
zap.String("old-group", removedMetaServiceGroup),
816+
zap.Error(err),
817+
)
818+
}
819+
}
810820
log.Info("[keyspace] keyspace state updated",
811821
zap.Uint32("keyspace-id", meta.GetId()),
812822
zap.String("name", meta.GetName()),
@@ -816,29 +826,25 @@ func (manager *Manager) UpdateKeyspaceStateByID(id uint32, newState keyspacepb.K
816826
}
817827

818828
// updateKeyspaceState updates keyspace meta and record the update time.
819-
func (manager *Manager) updateKeyspaceState(txn kv.Txn, meta *keyspacepb.KeyspaceMeta, newState keyspacepb.KeyspaceState, now int64) error {
829+
// It returns the removed meta-service group ID (if any) for post-commit updates.
830+
func (*Manager) updateKeyspaceState(meta *keyspacepb.KeyspaceMeta, newState keyspacepb.KeyspaceState, now int64) (string, error) {
831+
var removedMetaServiceGroup string
820832
if newState == keyspacepb.KeyspaceState_TOMBSTONE {
821-
oldMetaServiceGroup := meta.GetConfig()[MetaServiceGroupIDKey]
822-
removeMetaServiceGroup := ""
823-
// remove keyspace from meta-service group before set tombstone.
824-
err := manager.removeKeyspaceFromMSGroup(txn, meta, oldMetaServiceGroup, removeMetaServiceGroup)
825-
if err != nil {
826-
return err
827-
}
833+
removedMetaServiceGroup = meta.GetConfig()[MetaServiceGroupIDKey]
828834
}
829835

830836
// If already in the target state, do nothing and return.
831837
if meta.GetState() == newState {
832-
return nil
838+
return "", nil
833839
}
834840
// Consult state transition table to check if the operation is legal.
835841
if !slice.Contains(stateTransitionTable[meta.GetState()], newState) {
836-
return errors.Errorf("cannot change keyspace state from %s to %s", meta.GetState().String(), newState.String())
842+
return "", errors.Errorf("cannot change keyspace state from %s to %s", meta.GetState().String(), newState.String())
837843
}
838844
// If the operation is legal, update keyspace state and change time.
839845
meta.State = newState
840846
meta.StateChangedAt = now
841-
return nil
847+
return removedMetaServiceGroup, nil
842848
}
843849

844850
// LoadRangeKeyspace load up to limit keyspaces starting from keyspace with startID.

‎pkg/keyspace/keyspace_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ func (m *mockConfig) SetMetaServiceGroupsFallbackRatio(ratio float64) {
117117
}
118118

119119
func (suite *keyspaceTestSuite) SetupTest() {
120+
var err error
120121
re := suite.Require()
121122
suite.ctx, suite.cancel = context.WithCancel(context.Background())
122123
store := endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil)
@@ -126,9 +127,9 @@ func (suite *keyspaceTestSuite) SetupTest() {
126127
MetaServiceGroups: mockMetaServiceGroups(),
127128
}
128129
kgm := NewKeyspaceGroupManager(suite.ctx, store, nil)
129-
mgm := NewMetaServiceGroupManager(suite.ctx, store, &cfg)
130+
mgm, err := NewMetaServiceGroupManager(suite.ctx, store, &cfg)
131+
re.NoError(err)
130132
mustEnableMetaServiceGroups(re, mgm, mockMetaServiceGroups())
131-
var err error
132133
suite.manager, err = NewKeyspaceManager(suite.ctx, store, nil, allocator, &cfg, kgm, mgm)
133134
re.NoError(err)
134135
re.NoError(kgm.Bootstrap(suite.ctx))

0 commit comments

Comments
 (0)
⚡