fix(repo-server): Fix SystemCATrust for RHEL 9 (#1123)

The hardcoded image url caused the tests to pass despite the desired image have changed.

Signed-off-by: Oliver Gondža <ogondza@gmail.com>

Author: olivergondza

controllers/argocd/openshift/openshift.go

@@ -63,8 +63,7 @@ func ReconcilerHook(cr *argoapp.ArgoCD, v interface{}, hint string) error {
 		case cr.Name + "-repo-server":
 
 			prodImage := o.Spec.Template.Spec.Containers[0].Image
-			usingReleasedImages := strings.Contains(prodImage, "registry.redhat.io/openshift-gitops-1/argocd-rhel")
-			if cr.Spec.Repo.SystemCATrust != nil && usingReleasedImages {
+			if cr.Spec.Repo.SystemCATrust != nil {
 				updateSystemCATrustBuilding(cr, o, prodImage, logv)
 			}
 		}
@@ -154,7 +153,8 @@ done
 echo "User defined trusted CA files:"
 ls /etc/pki/ca-trust/source/anchors/
 
-update-ca-trust
+# Specifying the explicit location to turn on the container-aware behavior
+update-ca-trust extract --output /etc/pki/ca-trust/extracted
 
 echo "Trusted anchors:"
 trust list

test/openshift/e2e/ginkgo/fixture/clusterserviceversion/fixture.go

@@ -2,6 +2,7 @@ package clusterserviceversion
 
 import (
 	"context"
+	"strings"
 
 	. "github.com/onsi/gomega"
 	olmv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
@@ -27,5 +28,16 @@ func Update(obj *olmv1alpha1.ClusterServiceVersion, modify func(*olmv1alpha1.Clu
 		return k8sClient.Update(context.Background(), obj)
 	})
 	Expect(err).ToNot(HaveOccurred())
+}
 
+func Get(ctx context.Context, k8sClient client.Client) *olmv1alpha1.ClusterServiceVersion {
+	var csvList olmv1alpha1.ClusterServiceVersionList
+	Expect(k8sClient.List(ctx, &csvList, client.InNamespace("openshift-gitops-operator"))).To(Succeed())
+	for idx := range csvList.Items {
+		idxCSV := csvList.Items[idx]
+		if strings.Contains(idxCSV.Name, "gitops-operator") {
+			return &idxCSV
+		}
+	}
+	return nil
 }

test/openshift/e2e/ginkgo/fixture/utils/fixtureUtils.go

@@ -3,6 +3,7 @@ package utils
 import (
 	"os"
 
+	certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -146,6 +147,10 @@ func getKubeClient(config *rest.Config) (client.Client, *runtime.Scheme, error)
 		return nil, nil, err
 	}
 
+	if err := certificatesv1beta1.AddToScheme(scheme); err != nil {
+		return nil, nil, err
+	}
+
 	k8sClient, err := client.New(config, client.Options{Scheme: scheme})
 	if err != nil {
 		return nil, nil, err

test/openshift/e2e/ginkgo/sequential/1-085_validate_dynamic_plugin_installation_test.go

@@ -8,7 +8,7 @@ import (
 	. "github.com/onsi/gomega"
 	olmv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture"
-	clusterserviceversionFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/clusterserviceversion"
+	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/clusterserviceversion"
 	deploymentFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/deployment"
 	k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
 	osFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/os"
@@ -86,7 +86,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 
 			By("adding DYNAMIC_PLUGIN_START_OCP_VERSION to CSV operator Deployment env var list")
 
-			clusterserviceversionFixture.Update(csv, func(csv *olmv1alpha1.ClusterServiceVersion) {
+			clusterserviceversion.Update(csv, func(csv *olmv1alpha1.ClusterServiceVersion) {
 
 				envList := csv.Spec.InstallStrategy.StrategySpec.DeploymentSpecs[0].Spec.Template.Spec.Containers[0].Env
 				envList = append(envList, corev1.EnvVar{Name: "DYNAMIC_PLUGIN_START_OCP_VERSION", Value: ocVersion})

test/openshift/e2e/ginkgo/sequential/1-115_validate_imagepullpolicy_console_plugin_test.go

@@ -25,6 +25,7 @@ import (
 	gitopsoperatorv1alpha1 "github.com/redhat-developer/gitops-operator/api/v1alpha1"
 	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture"
 	argocdFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd"
+	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/clusterserviceversion"
 	gitopsserviceFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/gitopsservice"
 	k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
 	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/utils"
@@ -67,7 +68,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 			Eventually(argoCD).Should(k8sFixture.ExistByName())
 			Eventually(argoCD).Should(argocdFixture.BeAvailable())
 
-			csv := getCSV(ctx, k8sClient)
+			csv := clusterserviceversion.Get(ctx, k8sClient)
 			Expect(csv).ToNot(BeNil())
 			defer func() { Expect(fixture.RemoveDynamicPluginFromCSV(ctx, k8sClient)).To(Succeed()) }()
 
@@ -207,7 +208,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 			Eventually(argoCD).Should(k8sFixture.ExistByName())
 			Eventually(argoCD).Should(argocdFixture.BeAvailable())
 
-			csv := getCSV(ctx, k8sClient)
+			csv := clusterserviceversion.Get(ctx, k8sClient)
 			Expect(csv).ToNot(BeNil())
 			defer func() { Expect(fixture.RemoveDynamicPluginFromCSV(ctx, k8sClient)).To(Succeed()) }()
 
@@ -264,7 +265,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 		// 		return
 		// 	}
 
-		// 	csv := getCSV(ctx, k8sClient)
+		// 	csv := clusterserviceversion.Get(ctx, k8sClient)
 		// 	Expect(csv).ToNot(BeNil())
 		// 	defer func() { Expect(fixture.RemoveDynamicPluginFromCSV(ctx, k8sClient)).To(Succeed()) }()
 

test/openshift/e2e/ginkgo/sequential/1-120_repo_server_system_ca_trust.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/onsi/gomega/gcustom"
 	matcher "github.com/onsi/gomega/types"
+	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/clusterserviceversion"
 	"k8s.io/apimachinery/pkg/util/rand"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
@@ -51,15 +52,10 @@ import (
 	argov1beta1api "github.com/argoproj-labs/argocd-operator/api/v1beta1"
 	"github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture"
 	argocdFixture "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/argocd"
-	fixtureUtils "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/utils"
+	fixtureUtils "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/utils"
 )
 
 var (
-	// The differences between the upstream image using Ubuntu, and the downstream one using rhel.
-	image        = "" // argocd-operator default
-	imageVersion = "" // argocd-operator default
-	caBundlePath = "/etc/ssl/certs/ca-certificates.crt"
-
 	trustedHelmAppSource = &appv1alpha1.ApplicationSource{
 		RepoURL:        "https://stefanprodan.github.io/podinfo",
 		Chart:          "podinfo",
@@ -74,14 +70,17 @@ var (
 		Helm:           &appv1alpha1.ApplicationSourceHelm{Values: "service:\n          type: ClusterIP"},
 	}
 
-	k8sClient client.Client
-	ctx       context.Context
+	k8sClient    client.Client
+	ctx          context.Context
+	ns           *corev1.Namespace
+	cleanupNs    func()
+	image        string
+	imageVersion string
 
 	clusterSupportsClusterTrustBundles bool
 )
 
 var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
-
 	Context("1-120_repo_server_system_ca_trust", func() {
 		BeforeEach(func() {
 			fixture.EnsureSequentialCleanSlate()
@@ -91,24 +90,18 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 
 			clusterSupportsClusterTrustBundles = detectClusterTrustBundleSupport(k8sClient, ctx)
 
-			if fixture.EnvLocalRun() {
-				Skip("skipping test as LOCAL_RUN env is set.")
-			}
-
-			if !fixture.EnvNonOLM() {
-				image = "registry.redhat.io/openshift-gitops-1/argocd-rhel8"
-				imageVersion = "sha256:8a0544c14823492165550d83a6d8ba79dd632b46144d3fdcb543793726111d76"
-				caBundlePath = "/etc/ssl/certs/ca-bundle.crt"
-			}
+			// The differences between the upstream image using Ubuntu, and the downstream one using rhel.
+			image, imageVersion = getArgoCDComponentImageAndVersion()
 		})
 
 		AfterEach(func() {
+			fixture.OutputDebugOnFail(ns)
+			cleanupNs()
 			purgeCtbs()
 		})
 
 		It("ensures that missing Secret aborts startup", func() {
-			ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
-			defer cleanupFunc()
+			ns, cleanupNs = fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
 
 			By("creating Argo CD instance with missing Secret")
 			argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{
@@ -130,8 +123,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 				Skip("Cluster does not support ClusterTrustBundles")
 			}
 
-			ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
-			defer cleanupFunc()
+			ns, cleanupNs = fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
 
 			// Create a bundle with 2 CA certs in it. Ubuntu's update-ca-certificates issues a warning, but apparently it works
 			// It is desirable to test with multiple certs in one bundle because OpenShift permits it
@@ -171,8 +163,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 		})
 
 		It("ensures that CMs and Secrets are trusted in repo-server and plugins", func() {
-			ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
-			defer cleanupFunc()
+			ns, cleanupNs = fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
 
 			cmCert := createCmFromCert(ns, getCACert("github.com"))
 			Expect(k8sClient.Create(ctx, cmCert)).To(Succeed())
@@ -220,8 +211,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 		})
 
 		It("ensures that 0 trusted certs with DropImageCertificates trusts nothing", func() {
-			ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
-			defer cleanupFunc()
+			ns, cleanupNs = fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
 
 			By("creating Argo CD instance with empty system trust")
 			argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{
@@ -253,8 +243,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 		})
 
 		It("ensures that empty trust keeps image certs in place", func() {
-			ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
-			defer cleanupFunc()
+			ns, cleanupNs = fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
 
 			By("creating Argo CD instance with empty system trust")
 			argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{
@@ -268,8 +257,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 		})
 
 		It("ensures that Secrets and ConfigMaps get reconciled", func() {
-			ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
-			defer cleanupFunc()
+			ns, cleanupNs = fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
 
 			By("creating Argo CD instance with empty system trust, but full of anticipation")
 			argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{
@@ -360,8 +348,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 				Skip("Cluster does not support ClusterTrustBundles")
 			}
 
-			ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
-			defer cleanupFunc()
+			ns, cleanupNs = fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
 
 			combinedCtb := createCtbFromCerts(getCACert("github.com"), getCACert("github.io"))
 			_ = k8sClient.Delete(ctx, combinedCtb) // Exists only in case of previous failures, must be deleted before argo starts!
@@ -413,8 +400,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 				Skip("Cluster does not support ClusterTrustBundles")
 			}
 
-			ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
-			defer cleanupFunc()
+			ns, cleanupNs = fixture.CreateRandomE2ETestNamespaceWithCleanupFunc()
 
 			// Use random label value not to collide with leftover CTBs fom other tests
 			labelVal := rand.String(5)
@@ -804,7 +790,9 @@ func getTrustedCertCount(rsPod *corev1.Pod) int {
 	command := []string{
 		"kubectl", "-n", rsPod.Namespace, "exec",
 		"-c", "argocd-repo-server", rsPod.Name, "--",
-		"cat", caBundlePath,
+		"bash", "-c",
+		// Ubuntu or RHEL location
+		"cat /etc/ssl/certs/ca-certificates.crt || cat /etc/ssl/certs/ca-bundle.crt",
 	}
 
 	var out string
@@ -909,3 +897,22 @@ func purgeCtbs() {
 		Expect(k8sClient.DeleteAllOf(ctx, &certificatesv1beta1.ClusterTrustBundle{}, expr)).To(Succeed())
 	}
 }
+
+// getArgoCDComponentImageAndVersion ensures rhel based image is used for testing this.
+// If this is installed as a full-blown operator, use its images.
+// If not, inject bleeding-edge pre-release rhel-based images.
+func getArgoCDComponentImageAndVersion() (string, string) {
+	csv := clusterserviceversion.Get(ctx, k8sClient)
+	if csv != nil {
+		for _, image := range csv.Spec.RelatedImages {
+			if image.Name == "argocd_image" {
+				By("Detected operator image " + image.Image + ". Using it")
+				return "", ""
+			}
+		}
+	}
+
+	defImage := "quay.io/redhat-user-workloads/rh-openshift-gitops-tenant/argocd-rhel9"
+	By("Injecting rhel image " + defImage)
+	return defImage, "main"
+}

test/openshift/e2e/ginkgo/sequential/1-121-valiate_resource_constraints_gitopsservice_test.go

@@ -9,7 +9,7 @@ import (
 	olmv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	gitopsoperatorv1alpha1 "github.com/redhat-developer/gitops-operator/api/v1alpha1"
 	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture"
-	clusterserviceversionFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/clusterserviceversion"
+	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/clusterserviceversion"
 	deploymentFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/deployment"
 	gitopsserviceFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/gitopsservice"
 	k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
@@ -24,18 +24,6 @@ import (
 
 // --- Helper Functions ---
 
-func getCSV(ctx context.Context, k8sClient client.Client) *olmv1alpha1.ClusterServiceVersion {
-	var csvList olmv1alpha1.ClusterServiceVersionList
-	Expect(k8sClient.List(ctx, &csvList, client.InNamespace("openshift-gitops-operator"))).To(Succeed())
-	for idx := range csvList.Items {
-		idxCSV := csvList.Items[idx]
-		if strings.Contains(idxCSV.Name, "gitops-operator") {
-			return &idxCSV
-		}
-	}
-	return nil
-}
-
 func getOCPVersion() string {
 	output, err := osFixture.ExecCommand("oc", "version")
 	Expect(err).ToNot(HaveOccurred())
@@ -48,7 +36,7 @@ func getOCPVersion() string {
 }
 
 func addDynamicPluginEnv(csv *olmv1alpha1.ClusterServiceVersion, ocVersion string) {
-	clusterserviceversionFixture.Update(csv, func(csv *olmv1alpha1.ClusterServiceVersion) {
+	clusterserviceversion.Update(csv, func(csv *olmv1alpha1.ClusterServiceVersion) {
 		envList := csv.Spec.InstallStrategy.StrategySpec.DeploymentSpecs[0].Spec.Template.Spec.Containers[0].Env
 		envList = append(envList, corev1.EnvVar{Name: "DYNAMIC_PLUGIN_START_OCP_VERSION", Value: ocVersion})
 		csv.Spec.InstallStrategy.StrategySpec.DeploymentSpecs[0].Spec.Template.Spec.Containers[0].Env = envList
@@ -86,7 +74,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 		})
 
 		It("validates that GitOpsService can take in custom resource constraints", func() {
-			csv := getCSV(ctx, k8sClient)
+			csv := clusterserviceversion.Get(ctx, k8sClient)
 			Expect(csv).ToNot(BeNil())
 			defer func() { Expect(fixture.RemoveDynamicPluginFromCSV(ctx, k8sClient)).To(Succeed()) }()
 
@@ -167,7 +155,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 		})
 
 		It("validates that GitOpsService can update resource constraints", func() {
-			csv := getCSV(ctx, k8sClient)
+			csv := clusterserviceversion.Get(ctx, k8sClient)
 			Expect(csv).ToNot(BeNil())
 			defer func() { Expect(fixture.RemoveDynamicPluginFromCSV(ctx, k8sClient)).To(Succeed()) }()
 
@@ -238,7 +226,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 		})
 
 		It("validates gitops plugin and backend can have different resource constraints", func() {
-			csv := getCSV(ctx, k8sClient)
+			csv := clusterserviceversion.Get(ctx, k8sClient)
 			Expect(csv).ToNot(BeNil())
 			defer func() { Expect(fixture.RemoveDynamicPluginFromCSV(ctx, k8sClient)).To(Succeed()) }()