client: clean up demand RU gauge and clarify its semantics

Address review feedback on #10582:

- Delete `DemandRUPerSecGauge` label series in `cleanUpResourceGroup`
  so the new gauge does not leak labels when a resource group is
  deleted. Add a TODO tracking the remaining per-group metrics that
  still leak (TokenConsumedHistogram, GroupRunningKVRequestCounter,
  SuccessfulRequestDuration, FailedRequestCounter,
  ResourceGroupTokenRequestCounter, RequestRetryCounter,
  FailedLimitReserveDuration).
- Clarify the metric Help text and doc comment to make the
  pre-throttling semantics explicit: the EMA includes requests
  rejected by the token bucket, which is the whole reason this
  metric is exposed separately from the consumption-based
  `avg_ru_per_sec`.

Signed-off-by: JmPotato <github@ipotato.me>

Author: JmPotato

client/resource_group/controller/global_controller.go

@@ -634,6 +634,11 @@ func (c *ResourceGroupsController) cleanUpResourceGroup() {
 			if gc.inactive || gc.tombstone.Load() {
 				c.groupsController.Delete(resourceGroupName)
 				metrics.ResourceGroupStatusGauge.DeleteLabelValues(resourceGroupName, resourceGroupName)
+				metrics.DemandRUPerSecGauge.DeleteLabelValues(resourceGroupName)
+				// TODO: clean up the remaining per-group metrics (e.g. TokenConsumedHistogram,
+				// GroupRunningKVRequestCounter, SuccessfulRequestDuration, FailedRequestCounter,
+				// ResourceGroupTokenRequestCounter, RequestRetryCounter, FailedLimitReserveDuration)
+				// which currently leak label series on resource group deletion.
 				return true
 			}
 			gc.inactive = true

client/resource_group/controller/metrics/metrics.go

@@ -31,7 +31,8 @@ const (
 var (
 	// ResourceGroupStatusGauge comments placeholder
 	ResourceGroupStatusGauge *prometheus.GaugeVec
-	// DemandRUPerSecGauge is the EMA of demanded RU/s before throttling per resource group.
+	// DemandRUPerSecGauge is the EMA of demanded RU/s per resource group, including
+	// requests rejected by the token bucket (pre-throttling demand).
 	DemandRUPerSecGauge *prometheus.GaugeVec
 	// SuccessfulRequestDuration comments placeholder
 	SuccessfulRequestDuration *prometheus.HistogramVec
@@ -76,7 +77,7 @@ func initMetrics(constLabels prometheus.Labels) {
 			Namespace:   namespace,
 			Subsystem:   "resource_group",
 			Name:        "demand_ru_per_sec",
-			Help:        "EMA of demanded RU/s before throttling for each resource group.",
+			Help:        "EMA of demanded RU/s per resource group, including requests rejected by the token bucket (pre-throttling demand).",
 			ConstLabels: constLabels,
 		}, []string{newResourceGroupNameLabel})