test with adding dataplane using the KonnectExtension

Author: randmonkey

test/integration/test_konnect_extension.go

@@ -2,22 +2,30 @@ package integration
 
 import (
 	"fmt"
+	"net/http"
 	"testing"
+	"time"
 
 	"github.com/google/uuid"
+	"github.com/kong/kubernetes-testing-framework/pkg/utils/kubernetes/generators"
 	"github.com/samber/lo"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/types"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/kong/gateway-operator/pkg/consts"
 	testutils "github.com/kong/gateway-operator/pkg/utils/test"
 	"github.com/kong/gateway-operator/test"
 	"github.com/kong/gateway-operator/test/helpers"
 	"github.com/kong/gateway-operator/test/helpers/deploy"
 
 	kcfgconsts "github.com/kong/kubernetes-configuration/api/common/consts"
+	commonv1alpha1 "github.com/kong/kubernetes-configuration/api/common/v1alpha1"
+	configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1"
+	operatorv1beta1 "github.com/kong/kubernetes-configuration/api/gateway-operator/v1beta1"
 	konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1"
 )
 
@@ -50,7 +58,7 @@ func TestKonnectExtension(t *testing.T) {
 
 	t.Logf("Waiting for Konnect ID to be assigned to ControlPlane %s/%s", cp.Namespace, cp.Name)
 	require.EventuallyWithT(t, func(t *assert.CollectT) {
-		err := GetClients().MgrClient.Get(GetCtx(), types.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, cp)
+		err := GetClients().MgrClient.Get(GetCtx(), k8stypes.NamespacedName{Name: cp.Name, Namespace: cp.Namespace}, cp)
 		require.NoError(t, err)
 		assertKonnectEntityProgrammed(t, cp)
 	}, testutils.ObjectUpdateTimeout, testutils.ObjectUpdateTick)
@@ -92,6 +100,107 @@ func TestKonnectExtension(t *testing.T) {
 		checkKonnectExtensionStatus(keWithKonnectIDCPRef, cp.GetKonnectID(), dpCert1.Name),
 		testutils.ObjectUpdateTimeout, testutils.ObjectUpdateTick)
 
+	// Create a DataPlane using the KonnectExension.
+	t.Logf("Creating a DataPlane using the KonnectExtension %s/%s", keWithKonnectIDCPRef.Namespace, keWithKonnectIDCPRef.Name)
+	dp := &operatorv1beta1.DataPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: ns.String(),
+			Name:      "test-konnect-extension-dp-1",
+		},
+		Spec: operatorv1beta1.DataPlaneSpec{
+			DataPlaneOptions: operatorv1beta1.DataPlaneOptions{
+				Extensions: []commonv1alpha1.ExtensionRef{
+					{
+						Group: konnectv1alpha1.GroupVersion.Group,
+						Kind:  "KonnectExtension",
+						NamespacedRef: commonv1alpha1.NamespacedRef{
+							Name: keWithKonnectIDCPRef.Name,
+						},
+					},
+				},
+			},
+		},
+	}
+	require.NoError(t, clientNamespaced.Create(ctx, dp))
+
+	dpName := k8stypes.NamespacedName{
+		Namespace: dp.Namespace,
+		Name:      dp.Name,
+	}
+	t.Logf("verifying dataplane %s gets marked provisioned", dpName)
+
+	require.Eventually(t, testutils.DataPlaneIsReady(t, GetCtx(), dpName, GetClients().OperatorClient), waitTime, tickTime)
+
+	t.Logf("verifying dataplane %s has ingress service", dpName)
+	var dpIngressService corev1.Service
+	require.Eventually(t, testutils.DataPlaneHasActiveService(t, GetCtx(), dpName, &dpIngressService, clients, client.MatchingLabels{
+		consts.GatewayOperatorManagedByLabel: consts.DataPlaneManagedLabelValue,
+		consts.DataPlaneServiceTypeLabel:     string(consts.DataPlaneIngressServiceLabelValue),
+	}), waitTime, tickTime)
+
+	t.Log("verifying dataplane services receive IP addresses")
+	require.Eventually(t, func() bool {
+		err := clientNamespaced.Get(ctx, k8stypes.NamespacedName{
+			Namespace: dpIngressService.Namespace,
+			Name:      dpIngressService.Name,
+		}, &dpIngressService)
+		require.NoError(t, err)
+		return len(dpIngressService.Status.LoadBalancer.Ingress) > 0
+	}, waitTime, tickTime)
+	dpIngressIP := dpIngressService.Status.LoadBalancer.Ingress[0].IP
+	require.Eventuallyf(t, Expect404WithNoRouteFunc(t, GetCtx(), "http://"+dpIngressIP), waitTime, tickTime,
+		"Should receive 'No Route' response from dataplane's ingress service IP %s", dpIngressIP)
+
+	t.Log("deploying backend deployment (httpbin) of HTTPRoute")
+	container := generators.NewContainer("httpbin", testutils.HTTPBinImage, 80)
+	deployment := generators.NewDeploymentForContainer(container)
+	require.NoError(t, clientNamespaced.Create(ctx, deployment))
+
+	t.Logf("exposing deployment %s via service", deployment.Name)
+	service := generators.NewServiceForDeployment(deployment, corev1.ServiceTypeClusterIP)
+	require.NoError(t, clientNamespaced.Create(ctx, service))
+
+	t.Log("Creating a KongService and a KongRoute to the service")
+	ks := deploy.KongService(t, ctx, clientNamespaced,
+		deploy.WithKonnectNamespacedRefControlPlaneRef(cp),
+		func(obj client.Object) {
+			ks, ok := obj.(*configurationv1alpha1.KongService)
+			require.True(t, ok)
+			ks.Spec.KongServiceAPISpec = configurationv1alpha1.KongServiceAPISpec{
+				Name: lo.ToPtr("httpbin"),
+				URL:  lo.ToPtr(fmt.Sprintf("http://%s.%s.svc.cluster.local/", service.Name, ns.Name)),
+				Host: fmt.Sprintf("%s.%s.svc.cluster.local", service.Name, ns.Name),
+			}
+		},
+	)
+	t.Logf("Waiting for KongService to be updated with Konnect ID")
+	require.EventuallyWithT(t, func(t *assert.CollectT) {
+		err := GetClients().MgrClient.Get(GetCtx(), k8stypes.NamespacedName{Name: ks.Name, Namespace: ks.Namespace}, ks)
+		require.NoError(t, err)
+		assertKonnectEntityProgrammed(t, ks)
+	}, testutils.ObjectUpdateTimeout, testutils.ObjectUpdateTick)
+
+	kr := deploy.KongRouteAttachedToService(t, ctx, clientNamespaced, ks)
+	t.Logf("Waiting for KongRoute to be updated with Konnect ID")
+	require.EventuallyWithT(t, func(t *assert.CollectT) {
+		err := GetClients().MgrClient.Get(GetCtx(), k8stypes.NamespacedName{Name: kr.Name, Namespace: kr.Namespace}, kr)
+		require.NoError(t, err)
+
+		assertKonnectEntityProgrammed(t, kr)
+	}, testutils.ObjectUpdateTimeout, testutils.ObjectUpdateTick)
+
+	t.Log("route to / path of service httpbin should receive a 200 OK response")
+	httpClient, err := helpers.CreateHTTPClient(nil, "")
+	require.NoError(t, err)
+	const routeAccessTimeout = 3 * time.Minute
+	request := helpers.MustBuildRequest(t, GetCtx(), http.MethodGet, "http://"+dpIngressIP+"/test", "")
+	require.Eventually(
+		t,
+		testutils.GetResponseBodyContains(t, clients, httpClient, request, "<title>httpbin.org</title>"),
+		routeAccessTimeout,
+		time.Second,
+	)
+
 	// Tests on KonnectExtension with KonnectNamespacedRef control plane ref.
 	dpCert2 := deploy.Secret(
 		t, ctx, clientNamespaced,
@@ -122,9 +231,12 @@ func TestKonnectExtension(t *testing.T) {
 		testutils.ObjectUpdateTimeout, testutils.ObjectUpdateTick)
 
 	// Order of deleting objects with finalizers:
-	// KonnectExtension -> Secret -> KonnectGatewayControlPlane.
+	// DataPlane -> KonnectExtension -> Secret -> KonnectGatewayControlPlane.
 	// After they are all deleted, the namespace can be deleted in the final clean up.
+
 	t.Cleanup(deleteObjectAndWaitForDeletionFn(t, cp.DeepCopy()))
+	t.Cleanup(deleteObjectAndWaitForDeletionFn(t, ks))
+	t.Cleanup(deleteObjectAndWaitForDeletionFn(t, kr))
 	t.Cleanup(deleteObjectAndWaitForDeletionFn(t, dpCert1.DeepCopy()))
 	t.Cleanup(deleteObjectAndWaitForDeletionFn(t, dpCert2.DeepCopy()))
 	t.Cleanup(deleteObjectAndWaitForDeletionFn(t, keWithKonnectIDCPRef.DeepCopy()))
@@ -148,7 +260,7 @@ func setKonnectExtensionDPCertSecretRef(t *testing.T, s *corev1.Secret) deploy.O
 }
 
 func checkKonnectExtensionConditions(t *assert.CollectT, ke *konnectv1alpha1.KonnectExtension) (bool, string) {
-	err := GetClients().MgrClient.Get(GetCtx(), types.NamespacedName{Name: ke.Name, Namespace: ke.Namespace}, ke)
+	err := GetClients().MgrClient.Get(GetCtx(), k8stypes.NamespacedName{Name: ke.Name, Namespace: ke.Namespace}, ke)
 	require.NoError(t, err)
 
 	checkConditionTypes := []kcfgconsts.ConditionType{
@@ -165,7 +277,7 @@ func checkKonnectExtensionStatus(
 	expectedDPCertificateSecretName string,
 ) func(t *assert.CollectT) {
 	return func(t *assert.CollectT) {
-		err := GetClients().MgrClient.Get(GetCtx(), types.NamespacedName{Name: ke.Name, Namespace: ke.Namespace}, ke)
+		err := GetClients().MgrClient.Get(GetCtx(), k8stypes.NamespacedName{Name: ke.Name, Namespace: ke.Namespace}, ke)
 		require.NoError(t, err)
 		// Check Konnect control plane ID
 		require.NotNil(t, ke.Status.Konnect, "status.konnect should be present")