ruler: add restore-new-rule-groups flag

Author: MichaHoffmann

cmd/thanos/rule.go

@@ -101,18 +101,19 @@ type ruleConfig struct {
 
 	rwConfig *extflag.PathOrContent
 
-	resendDelay        time.Duration
-	evalInterval       time.Duration
-	queryOffset        time.Duration
-	outageTolerance    time.Duration
-	forGracePeriod     time.Duration
-	ruleFiles          []string
-	objStoreConfig     *extflag.PathOrContent
-	dataDir            string
-	lset               labels.Labels
-	ignoredLabelNames  []string
-	storeRateLimits    store.SeriesSelectLimits
-	ruleConcurrentEval int64
+	resendDelay          time.Duration
+	evalInterval         time.Duration
+	queryOffset          time.Duration
+	outageTolerance      time.Duration
+	forGracePeriod       time.Duration
+	ruleFiles            []string
+	objStoreConfig       *extflag.PathOrContent
+	dataDir              string
+	lset                 labels.Labels
+	ignoredLabelNames    []string
+	storeRateLimits      store.SeriesSelectLimits
+	ruleConcurrentEval   int64
+	restoreNewRuleGroups bool
 
 	extendedFunctionsEnabled   bool
 	EnableFeatures             []string
@@ -166,6 +167,8 @@ func registerRule(app *extkingpin.App) {
 	cmd.Flag("restore-ignored-label", "Label names to be ignored when restoring alerts from the remote storage. This is only used in stateless mode.").
 		StringsVar(&conf.ignoredLabelNames)
 	cmd.Flag("rule-concurrent-evaluation", "How many rules can be evaluated concurrently. Default is 1.").Default("1").Int64Var(&conf.ruleConcurrentEval)
+	cmd.Flag("restore-new-rule-groups", "Enable restoring the 'for' state of new rule groups added via config reload by querying ALERTS_FOR_STATE series.").
+		Default("false").BoolVar(&conf.restoreNewRuleGroups)
 
 	cmd.Flag("grpc-query-endpoint", "Addresses of Thanos gRPC query API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect Thanos API servers through respective DNS lookups.").
 		PlaceHolder("<endpoint>").StringsVar(&conf.grpcQueryEndpoints)
@@ -640,6 +643,7 @@ func runRule(
 			OutageTolerance:        conf.outageTolerance,
 			ForGracePeriod:         conf.forGracePeriod,
 			DefaultRuleQueryOffset: func() time.Duration { return conf.queryOffset },
+			RestoreNewRuleGroups:   conf.restoreNewRuleGroups,
 		}
 		if conf.ruleConcurrentEval > 1 {
 			managerOpts.MaxConcurrentEvals = conf.ruleConcurrentEval

test/e2e/e2ethanos/services.go

@@ -815,6 +815,7 @@ type RulerBuilder struct {
 	evalInterval         string
 	forGracePeriod       string
 	restoreIgnoredLabels []string
+	restoreNewRuleGroups bool
 	nativeHistograms     bool
 }
 
@@ -866,6 +867,11 @@ func (r *RulerBuilder) WithRestoreIgnoredLabels(labels ...string) *RulerBuilder
 	return r
 }
 
+func (r *RulerBuilder) WithRestoreNewRuleGroups() *RulerBuilder {
+	r.restoreNewRuleGroups = true
+	return r
+}
+
 func (r *RulerBuilder) WithNativeHistograms() *RulerBuilder {
 	r.nativeHistograms = true
 	return r
@@ -938,6 +944,10 @@ func (r *RulerBuilder) initRule(internalRuleDir string, queryCfg []clientconfig.
 		ruleArgs["--remote-write.config"] = string(rwCfgBytes)
 	}
 
+	if r.restoreNewRuleGroups {
+		ruleArgs["--restore-new-rule-groups"] = ""
+	}
+
 	if r.nativeHistograms {
 		ruleArgs["--tsdb.enable-native-histograms"] = ""
 	}