placement-rule: move placement rule initialization after LoadStores (#9616) (#9758)

close #9602

Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io>
Signed-off-by: okjiang <819421878@qq.com>

Co-authored-by: okJiang <819421878@qq.com>

Author: ti-chi-bot

client/http/interface.go

@@ -50,6 +50,7 @@ type Client interface {
 	GetStores(context.Context) (*StoresInfo, error)
 	GetStore(context.Context, uint64) (*StoreInfo, error)
 	SetStoreLabels(context.Context, int64, map[string]string) error
+	DeleteStoreLabel(ctx context.Context, storeID int64, labelKey string) error
 	/* Config-related interfaces */
 	GetConfig(context.Context) (map[string]any, error)
 	SetConfig(context.Context, map[string]any, ...float64) error
@@ -337,6 +338,19 @@ func (c *client) SetStoreLabels(ctx context.Context, storeID int64, storeLabels
 		WithBody(jsonInput))
 }
 
+// DeleteStoreLabel deletes the labels of a store.
+func (c *client) DeleteStoreLabel(ctx context.Context, storeID int64, labelKey string) error {
+	jsonInput, err := json.Marshal(labelKey)
+	if err != nil {
+		return errors.Trace(err)
+	}
+	return c.request(ctx, newRequestInfo().
+		WithName(deleteStoreLabelName).
+		WithURI(LabelByStoreID(storeID)).
+		WithMethod(http.MethodDelete).
+		WithBody(jsonInput))
+}
+
 // GetConfig gets the configurations.
 func (c *client) GetConfig(ctx context.Context) (map[string]any, error) {
 	var config map[string]any

client/http/request_info.go

@@ -40,6 +40,7 @@ const (
 	getStoresName                           = "GetStores"
 	getStoreName                            = "GetStore"
 	setStoreLabelsName                      = "SetStoreLabels"
+	deleteStoreLabelName                    = "DeleteStoreLabel"
 	getConfigName                           = "GetConfig"
 	setConfigName                           = "SetConfig"
 	getScheduleConfigName                   = "GetScheduleConfig"

server/cluster/cluster.go

@@ -272,12 +272,6 @@ func (c *RaftCluster) InitCluster(
 	c.keyspaceGroupManager = keyspaceGroupManager
 	c.hbstreams = hbstreams
 	c.ruleManager = placement.NewRuleManager(c.ctx, c.storage, c, c.GetOpts())
-	if c.opt.IsPlacementRulesEnabled() {
-		err := c.ruleManager.Initialize(c.opt.GetMaxReplicas(), c.opt.GetLocationLabels(), c.opt.GetIsolationLevel())
-		if err != nil {
-			return err
-		}
-	}
 	c.schedulingController = newSchedulingController(c.ctx, c.core, c.opt, c.ruleManager)
 	return nil
 }
@@ -304,6 +298,12 @@ func (c *RaftCluster) Start(s Server) error {
 	if cluster == nil {
 		return nil
 	}
+	if c.opt.IsPlacementRulesEnabled() {
+		err := c.ruleManager.Initialize(c.opt.GetMaxReplicas(), c.opt.GetLocationLabels(), c.opt.GetIsolationLevel())
+		if err != nil {
+			return err
+		}
+	}
 
 	c.regionLabeler, err = labeler.NewRegionLabeler(c.ctx, c.storage, regionLabelGCInterval)
 	if err != nil {

tests/integrations/client/http_client_test.go

@@ -348,6 +348,49 @@ func (suite *httpClientTestSuite) checkRule(mode mode, client pd.Client) {
 	err = client.SetPlacementRule(env.ctx, testRule)
 	re.NoError(err)
 	suite.checkRuleResult(re, env, client, testRule, 1, true)
+
+	// ***** Test placement rule failed passing check after transfer leader
+	// Transfer the leader to another store to ensure the PD follower
+	// exists stale store labels.
+	suite.transferLeader(env, client)
+	tranferLeaderRule := []*pd.GroupBundle{
+		{
+			ID: "test-transfer-leader",
+			Rules: []*pd.Rule{
+				{
+					GroupID:  "test-transfer-leader",
+					ID:       "readonly",
+					Role:     pd.Voter,
+					Count:    3,
+					StartKey: []byte{},
+					EndKey:   []byte{},
+					LabelConstraints: []pd.LabelConstraint{
+						{
+							Key:    "$mode",
+							Op:     pd.In,
+							Values: []string{"readonly"},
+						},
+					},
+				},
+			},
+		},
+	}
+	err = client.SetPlacementRuleBundles(env.ctx, tranferLeaderRule, true)
+	re.Error(err)
+	re.ErrorContains(err, "invalid rule content, rule 'readonly' from rule group 'test-transfer-leader' can not match any store")
+	storeID := suite.setStoreLabels(env.ctx, client, map[string]string{
+		"$mode": "readonly",
+	})
+	err = client.SetPlacementRuleBundles(env.ctx, tranferLeaderRule, true)
+	re.NoError(err)
+	suite.checkRuleResult(re, env, client, tranferLeaderRule[0].Rules[0], 1, true)
+
+	suite.transferLeader(env, client)
+	suite.checkRuleResult(re, env, client, tranferLeaderRule[0].Rules[0], 1, true)
+	re.NoError(client.DeleteStoreLabel(env.ctx, storeID, "$mode"))
+	store, err := client.GetStore(env.ctx, uint64(storeID))
+	re.NoError(err)
+	re.Empty(store.Store.Labels)
 }
 
 func (suite *httpClientTestSuite) checkRuleResult(
@@ -593,20 +636,25 @@ func (suite *httpClientTestSuite) TestSetStoreLabels() {
 }
 
 func (suite *httpClientTestSuite) checkSetStoreLabels(mode mode, client pd.Client) {
-	re := suite.Require()
 	env := suite.env[mode]
 
-	resp, err := client.GetStores(env.ctx)
+	suite.setStoreLabels(env.ctx, client, map[string]string{
+		"zone": "zone1",
+	})
+}
+
+func (suite *httpClientTestSuite) setStoreLabels(ctx context.Context, client pd.Client, storeLabels map[string]string) int64 {
+	re := suite.Require()
+
+	resp, err := client.GetStores(ctx)
 	re.NoError(err)
 	setStore := resp.Stores[0]
 	re.Empty(setStore.Store.Labels, nil)
-	storeLabels := map[string]string{
-		"zone": "zone1",
-	}
-	err = client.SetStoreLabels(env.ctx, 1, storeLabels)
+
+	err = client.SetStoreLabels(ctx, 1, storeLabels)
 	re.NoError(err)
 
-	resp, err = client.GetStores(env.ctx)
+	resp, err = client.GetStores(ctx)
 	re.NoError(err)
 	for _, store := range resp.Stores {
 		if store.Store.ID == setStore.Store.ID {
@@ -615,16 +663,20 @@ func (suite *httpClientTestSuite) checkSetStoreLabels(mode mode, client pd.Clien
 			}
 		}
 	}
+	return setStore.Store.ID
 }
 
 func (suite *httpClientTestSuite) TestTransferLeader() {
 	suite.RunTestInTwoModes(suite.checkTransferLeader)
 }
 
 func (suite *httpClientTestSuite) checkTransferLeader(mode mode, client pd.Client) {
-	re := suite.Require()
 	env := suite.env[mode]
+	suite.transferLeader(env, client)
+}
 
+func (suite *httpClientTestSuite) transferLeader(env *httpClientTestEnv, client pd.Client) {
+	re := suite.Require()
 	members, err := client.GetMembers(env.ctx)
 	re.NoError(err)
 	re.Len(members.Members, 2)