tso fallback add metrics (tikv#431)

* add metrics

Signed-off-by: ystaticy <y_static_y@sina.com>

* add metrics

Signed-off-by: ystaticy <y_static_y@sina.com>

---------

Signed-off-by: ystaticy <y_static_y@sina.com>

Author: ystaticy

pkg/tso/keyspace_group_manager.go

@@ -316,13 +316,17 @@ func (s *state) getKeyspaceGroupMetaWithCheck(
 	// Before fallback to default group, check if the keyspace has configured a group in its metadata.
 	// If it has, don't fallback to avoid incorrect switching when state is inconsistent (e.g., after split).
 	if kgm != nil && checkKeyspaceHasConfiguredGroup(kgm.legacySvcStorage, keyspaceID) {
-		log.Info("[tso] keyspace has configured group but not found in lookup table, skip fallback to default group",
+		kgm.metrics.keyspaceFallbackRejectedCounter.Inc()
+		log.Debug("[tso] keyspace has configured group but not found in lookup table, skip fallback to default group",
 			zap.Uint32("keyspace-id", keyspaceID),
 			zap.Uint32("requested-group-id", keyspaceGroupID))
 		return nil, nil, keyspaceGroupID, errs.ErrKeyspaceNotAssigned.FastGenByArgs(keyspaceID)
 	}
-	// Legacy keyspace without group configuration, fallback to default group.
-	log.Info("[tso] keyspace not found in any group, fallback to default group for legacy keyspace",
+	// The keyspace without group configuration, fallback to default group.
+	if kgm != nil {
+		kgm.metrics.keyspaceFallbackToDefaultCounter.Inc()
+	}
+	log.Debug("[tso] keyspace not found in any group, fallback to default group for legacy keyspace",
 		zap.Uint32("keyspace-id", keyspaceID),
 		zap.Uint32("requested-group-id", keyspaceGroupID))
 	return s.ams[constant.DefaultKeyspaceGroupID],

pkg/tso/metrics.go

@@ -84,6 +84,14 @@ var (
 			Help:      "Bucketed histogram of processing time(s) of the Keyspace Group operations.",
 			Buckets:   prometheus.ExponentialBuckets(0.0005, 2, 13),
 		}, []string{typeLabel})
+
+	keyspaceFallbackCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: tsoNamespace,
+			Subsystem: "keyspace",
+			Name:      "fallback_events",
+			Help:      "Counter of keyspace fallback events",
+		}, []string{typeLabel})
 )
 
 func init() {
@@ -94,6 +102,7 @@ func init() {
 	prometheus.MustRegister(tsoAllocatorRole)
 	prometheus.MustRegister(keyspaceGroupStateGauge)
 	prometheus.MustRegister(keyspaceGroupOpDuration)
+	prometheus.MustRegister(keyspaceFallbackCounter)
 }
 
 type tsoMetrics struct {
@@ -179,6 +188,9 @@ type keyspaceGroupMetrics struct {
 	finishSplitDuration     prometheus.Observer
 	finishMergeSendDuration prometheus.Observer
 	finishMergeDuration     prometheus.Observer
+
+	keyspaceFallbackRejectedCounter  prometheus.Counter
+	keyspaceFallbackToDefaultCounter prometheus.Counter
 }
 
 func newKeyspaceGroupMetrics() *keyspaceGroupMetrics {
@@ -193,5 +205,8 @@ func newKeyspaceGroupMetrics() *keyspaceGroupMetrics {
 		finishSplitDuration:     keyspaceGroupOpDuration.WithLabelValues("finish-split"),
 		finishMergeSendDuration: keyspaceGroupOpDuration.WithLabelValues("finish-merge-send"),
 		finishMergeDuration:     keyspaceGroupOpDuration.WithLabelValues("finish-merge"),
+
+		keyspaceFallbackRejectedCounter:  keyspaceFallbackCounter.WithLabelValues("rejected"),
+		keyspaceFallbackToDefaultCounter: keyspaceFallbackCounter.WithLabelValues("to_default"),
 	}
 }