Fix monitoring reloading tests

Ensure we verify the policy update was actually applied before querying
the monitoring endpoint. Add a retry on said request as well.

Author: swiatekm

testing/integration/ess/monitoring_probe_preserve_text_cfg_test.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/gofrs/uuid/v5"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
@@ -67,6 +68,7 @@ type MonitoringTextRunner struct {
 	suite.Suite
 	info         *define.Info
 	agentFixture *atesting.Fixture
+	agentID      string
 
 	ESHost string
 
@@ -121,11 +123,12 @@ func (runner *MonitoringTextRunner) SetupSuite() {
 	err = runner.agentFixture.WriteFileToWorkDir(ctx, defaultTextCfg, "elastic-agent.yml")
 	require.NoError(runner.T(), err)
 
-	policyResp, _, err := tools.InstallAgentWithPolicy(ctx, runner.T(), installOpts, runner.agentFixture, runner.info.KibanaClient, basePolicy)
+	policyResp, agentID, err := tools.InstallAgentWithPolicy(ctx, runner.T(), installOpts, runner.agentFixture, runner.info.KibanaClient, basePolicy)
 	require.NoError(runner.T(), err)
 
 	runner.policyID = policyResp.ID
 	runner.policyName = basePolicy.Name
+	runner.agentID = agentID
 
 	_, err = tools.InstallPackageFromDefaultFile(ctx, runner.info.KibanaClient, "system",
 		integration.PreinstalledPackages["system"], "testdata/system_integration_setup.json", uuid.Must(uuid.NewV4()).String(), policyResp.ID)
@@ -137,6 +140,8 @@ func (runner *MonitoringTextRunner) TestMonitoringLiveness() {
 	defer cancel()
 
 	runner.AllComponentsHealthy(ctx)
+	agent, err := runner.info.KibanaClient.GetAgent(ctx, kibana.GetAgentRequest{ID: runner.agentID})
+	require.NoError(runner.T(), err)
 
 	client := http.Client{Timeout: time.Second * 4}
 	endpoint := "http://localhost:6791/processes"
@@ -173,12 +178,21 @@ func (runner *MonitoringTextRunner) TestMonitoringLiveness() {
 	require.NoError(runner.T(), err)
 	require.Equal(runner.T(), http.StatusOK, statusCode, "non-200 status code; got response: %s", string(overrideResp))
 
+	// verify the new policy revision was applied
+	newPolicyRevision := agent.PolicyRevision + 1
+	require.Eventually(
+		runner.T(),
+		tools.IsPolicyRevision(ctx, runner.T(), runner.info.KibanaClient, runner.policyID, newPolicyRevision),
+		5*time.Minute, time.Second)
+
 	runner.AllComponentsHealthy(ctx)
 
 	updatedEndpoint := "http://localhost:6792/processes"
 	// second stage: ensure the HTTP config has updated
-	req, err = http.NewRequestWithContext(ctx, "GET", updatedEndpoint, nil)
-	require.NoError(runner.T(), err)
+	require.EventuallyWithT(runner.T(), func(collect *assert.CollectT) {
+		req, err = http.NewRequestWithContext(ctx, "GET", updatedEndpoint, nil)
+		require.NoError(collect, err)
+	}, time.Minute, time.Second)
 
 	initResp, err = client.Do(req)
 	require.NoError(runner.T(), err)

testing/integration/ess/monitoring_probe_reload_test.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/gofrs/uuid/v5"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
@@ -31,6 +32,7 @@ type MonitoringRunner struct {
 	suite.Suite
 	info         *define.Info
 	agentFixture *atesting.Fixture
+	agentID      string
 
 	ESHost string
 
@@ -81,11 +83,12 @@ func (runner *MonitoringRunner) SetupSuite() {
 	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
 	defer cancel()
 
-	policyResp, _, err := tools.InstallAgentWithPolicy(ctx, runner.T(), installOpts, runner.agentFixture, runner.info.KibanaClient, basePolicy)
+	policyResp, agentID, err := tools.InstallAgentWithPolicy(ctx, runner.T(), installOpts, runner.agentFixture, runner.info.KibanaClient, basePolicy)
 	require.NoError(runner.T(), err)
 
 	runner.policyID = policyResp.ID
 	runner.policyName = basePolicy.Name
+	runner.agentID = agentID
 
 	_, err = tools.InstallPackageFromDefaultFile(ctx, runner.info.KibanaClient, "system",
 		integration.PreinstalledPackages["system"], "testdata/system_integration_setup.json", uuid.Must(uuid.NewV4()).String(), policyResp.ID)
@@ -97,6 +100,8 @@ func (runner *MonitoringRunner) TestMonitoringLiveness() {
 	defer cancel()
 
 	runner.AllComponentsHealthy(ctx)
+	agent, err := runner.info.KibanaClient.GetAgent(ctx, kibana.GetAgentRequest{ID: runner.agentID})
+	require.NoError(runner.T(), err)
 
 	client := http.Client{Timeout: time.Second * 4}
 	endpoint := "http://localhost:6792/liveness"
@@ -133,11 +138,20 @@ func (runner *MonitoringRunner) TestMonitoringLiveness() {
 	require.NoError(runner.T(), err)
 	require.Equal(runner.T(), http.StatusOK, statusCode, "non-200 status code; got response: %s", string(overrideResp))
 
+	// verify the new policy revision was applied
+	newPolicyRevision := agent.PolicyRevision + 1
+	require.Eventually(
+		runner.T(),
+		tools.IsPolicyRevision(ctx, runner.T(), runner.info.KibanaClient, runner.agentID, newPolicyRevision),
+		5*time.Minute, time.Second)
+
 	runner.AllComponentsHealthy(ctx)
 
 	// check to make sure that we now have a liveness probe response
-	req, err = http.NewRequestWithContext(ctx, "GET", endpoint, nil)
-	require.NoError(runner.T(), err)
+	require.EventuallyWithT(runner.T(), func(collect *assert.CollectT) {
+		req, err = http.NewRequestWithContext(ctx, "GET", endpoint, nil)
+		require.NoError(collect, err)
+	}, time.Minute, time.Second)
 
 	// second check: the /liveness endpoint should now be responding
 	runner.CheckResponse(ctx, endpoint)