add metrcis

Signed-off-by: yisaer <disxiaofei@163.com>

Author: Yisaer

server/schedule/filter/filters.go

@@ -686,6 +686,8 @@ const (
 	EngineKey = "engine"
 	// EngineTiFlash is the tiflash value of the engine label.
 	EngineTiFlash = "tiflash"
+	// EngineTiKV indicates the tikv engine in metrics
+	EngineTiKV = "tikv"
 )
 
 var allSpecialUses = []string{SpecialUseHotRegion, SpecialUseReserved}

server/schedule/metrics.go

@@ -73,6 +73,22 @@ var (
 			Name:      "store_limit_cost",
 			Help:      "limit rate cost of store.",
 		}, []string{"store", "limit_type"})
+
+	scatterCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "pd",
+			Subsystem: "schedule",
+			Name:      "scatter_operators_count",
+			Help:      "Counter of region scatter operators.",
+		}, []string{"type", "event"})
+
+	scatterDistributionCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "pd",
+			Subsystem: "schedule",
+			Name:      "scatter_distribution",
+			Help:      "Counter of the distribution in scatter.",
+		}, []string{"store", "is_leader", "engine"})
 )
 
 func init() {
@@ -83,4 +99,6 @@ func init() {
 	prometheus.MustRegister(storeLimitRateGauge)
 	prometheus.MustRegister(storeLimitCostCounter)
 	prometheus.MustRegister(operatorWaitCounter)
+	prometheus.MustRegister(scatterCounter)
+	prometheus.MustRegister(scatterDistributionCounter)
 }

server/schedule/region_scatterer.go

@@ -137,6 +137,7 @@ const maxRetryLimit = 30
 func (r *RegionScatterer) ScatterRegionsByRange(startKey, endKey []byte, group string, retryLimit int) ([]*operator.Operator, map[uint64]error, error) {
 	regions := r.cluster.ScanRegions(startKey, endKey, -1)
 	if len(regions) < 1 {
+		scatterCounter.WithLabelValues("skip", "empty_region").Inc()
 		return nil, nil, errors.New("empty region")
 	}
 	failures := make(map[uint64]error, len(regions))
@@ -155,13 +156,16 @@ func (r *RegionScatterer) ScatterRegionsByRange(startKey, endKey []byte, group s
 // ScatterRegionsByID directly scatter regions by ScatterRegions
 func (r *RegionScatterer) ScatterRegionsByID(regionsID []uint64, group string, retryLimit int) ([]*operator.Operator, map[uint64]error, error) {
 	if len(regionsID) < 1 {
+		scatterCounter.WithLabelValues("skip", "empty_region").Inc()
 		return nil, nil, errors.New("empty region")
 	}
 	failures := make(map[uint64]error, len(regionsID))
 	var regions []*core.RegionInfo
 	for _, id := range regionsID {
 		region := r.cluster.GetRegion(id)
 		if region == nil {
+			scatterCounter.WithLabelValues("skip", "no_region").Inc()
+			log.Warn("failed to find region during scatter", zap.Uint64("region-id", id))
 			failures[id] = errors.New(fmt.Sprintf("failed to find region %v", id))
 			continue
 		}
@@ -187,6 +191,7 @@ func (r *RegionScatterer) ScatterRegionsByID(regionsID []uint64, group string, r
 // and the value of the failures indicates the failure error.
 func (r *RegionScatterer) ScatterRegions(regions map[uint64]*core.RegionInfo, failures map[uint64]error, group string, retryLimit int) ([]*operator.Operator, error) {
 	if len(regions) < 1 {
+		scatterCounter.WithLabelValues("skip", "empty_region").Inc()
 		return nil, errors.New("empty region")
 	}
 	if retryLimit > maxRetryLimit {
@@ -226,14 +231,20 @@ func (r *RegionScatterer) ScatterRegions(regions map[uint64]*core.RegionInfo, fa
 func (r *RegionScatterer) Scatter(region *core.RegionInfo, group string) (*operator.Operator, error) {
 	if !opt.IsRegionReplicated(r.cluster, region) {
 		r.cluster.AddSuspectRegions(region.GetID())
+		scatterCounter.WithLabelValues("skip", "not_replicated").Inc()
+		log.Warn("region not replicated during scatter", zap.Uint64("region-id", region.GetID()))
 		return nil, errors.Errorf("region %d is not fully replicated", region.GetID())
 	}
 
 	if region.GetLeader() == nil {
+		scatterCounter.WithLabelValues("skip", "no_leader").Inc()
+		log.Warn("region no leader during scatter", zap.Uint64("region-id", region.GetID()))
 		return nil, errors.Errorf("region %d has no leader", region.GetID())
 	}
 
 	if r.cluster.IsRegionHot(region) {
+		scatterCounter.WithLabelValues("skip", "hot").Inc()
+		log.Warn("region too hot during scatter", zap.Uint64("region-id", region.GetID()))
 		return nil, errors.Errorf("region %d is hot", region.GetID())
 	}
 
@@ -286,15 +297,19 @@ func (r *RegionScatterer) scatterRegion(region *core.RegionInfo, group string) *
 
 	op, err := operator.CreateScatterRegionOperator("scatter-region", r.cluster, region, targetPeers, targetLeader)
 	if err != nil {
+		scatterCounter.WithLabelValues("fail", "").Inc()
 		for _, peer := range region.GetPeers() {
 			targetPeers[peer.GetStoreId()] = peer
 		}
 		r.Put(targetPeers, region.GetLeader().GetStoreId(), group)
 		log.Debug("fail to create scatter region operator", errs.ZapError(err))
 		return nil
 	}
-	r.Put(targetPeers, targetLeader, group)
-	op.SetPriorityLevel(core.HighPriority)
+	if op != nil {
+		scatterCounter.WithLabelValues("success", "").Inc()
+		r.Put(targetPeers, targetLeader, group)
+		op.SetPriorityLevel(core.HighPriority)
+	}
 	return op
 }
 
@@ -375,10 +390,22 @@ func (r *RegionScatterer) Put(peers map[uint64]*metapb.Peer, leaderStoreID uint6
 		store := r.cluster.GetStore(storeID)
 		if ordinaryFilter.Target(r.cluster.GetOpts(), store) {
 			r.ordinaryEngine.selectedPeer.Put(storeID, group)
+			scatterDistributionCounter.WithLabelValues(
+				fmt.Sprintf("%v", storeID),
+				fmt.Sprintf("%v", false),
+				filter.EngineTiKV).Inc()
 		} else {
 			engine := store.GetLabelValue(filter.EngineKey)
 			r.specialEngines[engine].selectedPeer.Put(storeID, group)
+			scatterDistributionCounter.WithLabelValues(
+				fmt.Sprintf("%v", storeID),
+				fmt.Sprintf("%v", false),
+				engine).Inc()
 		}
 	}
 	r.ordinaryEngine.selectedLeader.Put(leaderStoreID, group)
+	scatterDistributionCounter.WithLabelValues(
+		fmt.Sprintf("%v", leaderStoreID),
+		fmt.Sprintf("%v", true),
+		filter.EngineTiKV).Inc()
 }