region_scatterer: fix the bug that could generate schedule with too many peers (#5920) (#5929)

ref #4570, close #5909, ref #5920

Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io>
Signed-off-by: Cabinfever_B <cabinfeveroier@gmail.com>

Co-authored-by: 混沌DM <hundundm@gmail.com>
Co-authored-by: Cabinfever_B <cabinfeveroier@gmail.com>

Author: 3 people

server/schedule/region_scatterer.go

@@ -320,6 +320,7 @@ func (r *RegionScatterer) scatterRegion(region *core.RegionInfo, group string) *
 				// it is considered that the selected peer select itself.
 				// This origin peer re-selects.
 				if _, ok := peers[newPeer.GetStoreId()]; !ok || peer.GetStoreId() == newPeer.GetStoreId() {
+					selectedStores[peer.GetStoreId()] = struct{}{}
 					break
 				}
 			}

server/schedule/region_scatterer_test.go

@@ -607,7 +607,7 @@ func TestRegionHasLearner(t *testing.T) {
 	checkLeader(scatterer.ordinaryEngine.selectedLeader)
 }
 
-// TestSelectedStores tests if the peer count has changed due to the picking strategy.
+// TestSelectedStoresTooFewPeers tests if the peer count has changed due to the picking strategy.
 // Ref https://github.com/tikv/pd/issues/4565
 func (s *testScatterRegionSuite) TestSelectedStores(c *C) {
 	ctx, cancel := context.WithCancel(context.Background())
@@ -643,6 +643,43 @@ func (s *testScatterRegionSuite) TestSelectedStores(c *C) {
 	}
 }
 
+// TestSelectedStoresTooManyPeers tests if the peer count has changed due to the picking strategy.
+// Ref https://github.com/tikv/pd/issues/5909
+func TestSelectedStoresTooManyPeers(t *testing.T) {
+	re := require.New(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	opt := config.NewTestOptions()
+	tc := mockcluster.NewCluster(ctx, opt)
+	stream := hbstream.NewTestHeartbeatStreams(ctx, tc.ID, tc, false)
+	oc := NewOperatorController(ctx, tc, stream)
+	// Add 4 stores.
+	for i := uint64(1); i <= 5; i++ {
+		tc.AddRegionStore(i, 0)
+		// prevent store from being disconnected
+		tc.SetStoreLastHeartbeatInterval(i, -10*time.Minute)
+	}
+	group := "group"
+	scatterer := NewRegionScatterer(ctx, tc, oc)
+	// priority 4 > 1 > 5 > 2 == 3
+	for i := 0; i < 1200; i++ {
+		scatterer.ordinaryEngine.selectedPeer.Put(2, group)
+		scatterer.ordinaryEngine.selectedPeer.Put(3, group)
+	}
+	for i := 0; i < 800; i++ {
+		scatterer.ordinaryEngine.selectedPeer.Put(5, group)
+	}
+	for i := 0; i < 400; i++ {
+		scatterer.ordinaryEngine.selectedPeer.Put(1, group)
+	}
+	// test region with peer 1 2 3
+	for i := uint64(1); i < 20; i++ {
+		region := tc.AddLeaderRegion(i+200, i%3+1, (i+1)%3+1, (i+2)%3+1)
+		op := scatterer.scatterRegion(region, group)
+		re.False(isPeerCountChanged(op))
+	}
+}
+
 func isPeerCountChanged(op *operator.Operator) bool {
 	if op == nil {
 		return false