tests: verify that metrics endpoints have metrics

Author: gaelgatelement

poetry.lock

@@ -37,6 +37,7 @@ types-pytz = "^2025.2.0.20250516"
 mypy = "^1.16.1"
 types-pyyaml = "^6.0.12.20250516"
 semver = "^3.0.4"
+prometheus-client = "^0.22.1"
 
 [build-system]
 requires = ["poetry-core"]

pyproject.toml

@@ -6,8 +6,16 @@
 import time
 
 import pyhelm3
+from lightkube import AsyncClient
+from lightkube.models.core_v1 import (
+    Capabilities,
+    Container,
+    PodSpec,
+    SeccompProfile,
+    SecurityContext,
+)
 from lightkube.models.meta_v1 import ObjectMeta
-from lightkube.resources.core_v1 import ConfigMap, Endpoints, Namespace, Secret
+from lightkube.resources.core_v1 import ConfigMap, Endpoints, Namespace, Pod, Secret
 
 from ..artifacts import CertKey, generate_cert
 from .utils import merge
@@ -98,3 +106,50 @@ async def get_deployment_marker(kube_client, generated_data, marker: str):
         name=f"{generated_data.release_name}-markers",
     )
     return configmap.data.get(marker)
+
+
+async def run_pod_with_args(kube_client: AsyncClient, namespace, image_name, pod_name, args):
+    pod = Pod(
+        metadata=ObjectMeta(name=pod_name + "-" + str(int(time.time()*1000)), namespace=namespace),
+        spec=PodSpec(restartPolicy="Never",
+                      containers=[Container(name="cmd", image=image_name, args=args,
+                                    securityContext=SecurityContext(
+                                    seccompProfile=SeccompProfile(type="RuntimeDefault"),
+                                    capabilities=Capabilities(drop=["ALL"]),
+                                    readOnlyRootFilesystem=True,
+                                    allowPrivilegeEscalation=False,
+                                    runAsNonRoot=True,
+                                    runAsUser=3000,
+                                    runAsGroup=3000,
+                                    ),
+                                )
+                            ])
+            )
+    try:
+        await kube_client.create(pod)
+        start_time = time.time()
+        now = time.time()
+        completed = False
+        while start_time + 30 > now and not completed:
+            found_pod = await kube_client.get(Pod, name=pod.metadata.name, namespace=pod.metadata.namespace)
+            if (found_pod.status.containerStatuses
+                and found_pod.status.containerStatuses[0].lastState
+                and found_pod.status.containerStatuses[0].lastState.terminated
+                and found_pod.status.containerStatuses[0].lastState.terminated.reason == "Completed"):
+                completed = True
+            else:
+                now = time.time()
+                await asyncio.sleep(1)
+        else:
+            if start_time + 30 > now:
+                raise RuntimeError(f"Pod {pod.metadata.name} did not start in time "
+                               f"(failed after {time.time() - now} seconds), "
+                               f"pod status: {found_pod.status}")
+
+        log_lines = ""
+        async for log_line in kube_client.log(pod.metadata.name, namespace=pod.metadata.namespace, 
+                                              container="cmd"):
+            log_lines += log_line
+        return log_lines
+    finally:
+        await kube_client.delete(Pod, name=pod.metadata.name, namespace=namespace)

tests/integration/lib/helpers.py

@@ -9,9 +9,10 @@
 from lightkube import AsyncClient
 from lightkube import operators as op
 from lightkube.resources.core_v1 import Pod, Service
+from prometheus_client.parser import text_string_to_metric_families
 
 from .fixtures.data import ESSData
-from .lib.helpers import wait_for_endpoint_ready
+from .lib.helpers import run_pod_with_args, wait_for_endpoint_ready
 from .lib.utils import read_service_monitor_kind
 
 
@@ -162,3 +163,58 @@ async def test_pods_monitored(
     assert all_monitorable_pods == monitored_pods, (
         f"Some pods are not monitored : {', '.join(list(set(all_monitorable_pods) ^ set(monitored_pods)))}"
     )
+
+
+
+@pytest.mark.skipif(
+    os.environ.get("SKIP_SERVICE_MONITORS_CRDS", "false") == "true", reason="ServiceMonitors not deployed"
+)
+@pytest.mark.asyncio_cooperative
+@pytest.mark.usefixtures("matrix_stack")
+async def test_service_monitors_point_to_metrics(
+    kube_client: AsyncClient,
+    generated_data: ESSData,
+):
+    async for service_monitor in kube_client.list(
+        await read_service_monitor_kind(kube_client),
+        namespace=generated_data.ess_namespace,
+        labels={"app.kubernetes.io/part-of": op.in_(["matrix-stack"])},
+    ):
+        async for service in kube_client.list(
+            Service, namespace=generated_data.ess_namespace, labels=service_monitor["spec"]["selector"]["matchLabels"]
+        ):
+            assert service.metadata, f"Encountered a service without metadata : {service}"
+            assert service.spec, f"Encountered a service without spec : {service}"
+            assert service.spec.ports, f"Ecountered a service without port : {service}"
+            assert service.spec.selector, f"Ecountered a service without selectors : {service}"
+
+            for endpoint in service_monitor["spec"]["endpoints"]:
+                service_port_names = [port.name for port in service.spec.ports if port.name]
+                if endpoint["port"] in service_port_names:
+                    break
+            # This Service does not have the named port. Potentially there's another Service that covers it
+            else:
+                continue
+        assert await has_actual_metrics_on_endpoint(
+            kube_client, generated_data, service, service_monitor["spec"]["endpoints"]
+        )
+
+
+async def has_actual_metrics_on_endpoint(
+    kube_client: AsyncClient, generated_data: ESSData, service: Service, endpoints
+):
+    found_metrics = False
+    for endpoint in endpoints:
+        for port_spec in service.spec.ports:
+            if port_spec.name == endpoint["port"]:
+                metrics_data = await run_pod_with_args(
+                    kube_client,
+                    generated_data.ess_namespace,
+                    "curlimages/curl:latest",
+                    "curl",
+                    ["-s", f"http://{service.metadata.name}.{generated_data.ess_namespace}.svc.cluster.local:{port_spec.port}/metrics"],
+                )
+                for metric_family in text_string_to_metric_families(metrics_data):
+                    assert metric_family.name, "Metric family has no name"
+                found_metrics = True
+    return found_metrics