wait for all replicasets to be ready

Author: gaelgatelement

newsfragments/629.internal.md

@@ -0,0 +1 @@
+In tests, wait for all replicasets to be ready before checking service endpoints and monitored pods.

tests/integration/lib/helpers.py

@@ -15,6 +15,7 @@
     SecurityContext,
 )
 from lightkube.models.meta_v1 import ObjectMeta
+from lightkube.resources.apps_v1 import ReplicaSet
 from lightkube.resources.core_v1 import ConfigMap, Endpoints, Namespace, Pod, Secret
 
 from ..artifacts import CertKey
@@ -166,3 +167,19 @@ async def run_pod_with_args(kube_client: AsyncClient, namespace, image_name, pod
         return log_lines
     finally:
         await kube_client.delete(Pod, name=pod.metadata.name, namespace=namespace)
+
+
+async def wait_for_all_replicas_ready(kube_client: AsyncClient, namespace: str):
+    timeout = 30
+    start_time = time.time()
+    now = time.time()
+    while start_time + timeout > now:
+        now = time.time()
+        await asyncio.sleep(0.5)
+        async for rs in kube_client.list(ReplicaSet, namespace=namespace):
+            if not rs.status or not rs.status.readyReplicas:
+                break
+            if rs.status.readyReplicas != rs.status.replicas:
+                break
+        else:
+            return

tests/integration/test_networking.py

@@ -12,7 +12,7 @@
 from prometheus_client.parser import text_string_to_metric_families
 
 from .fixtures.data import ESSData
-from .lib.helpers import run_pod_with_args, wait_for_endpoint_ready
+from .lib.helpers import run_pod_with_args, wait_for_all_replicaset_replicas_ready, wait_for_endpoint_ready
 from .lib.utils import read_service_monitor_kind
 
 
@@ -73,6 +73,9 @@ async def test_services_have_endpoints(
     kube_client: AsyncClient,
     generated_data: ESSData,
 ):
+    # Helm will stop waiting when 1 of replicas are ready
+    # We need to wait for all replicas to be ready to check that services all have endpoints
+    await wait_for_all_replicaset_replicas_ready(kube_client, generated_data.ess_namespace)
     endpoints_to_wait = []
     services = {}
     async for service in kube_client.list(
@@ -117,6 +120,9 @@ async def test_pods_monitored(
     kube_client: AsyncClient,
     generated_data: ESSData,
 ):
+    # Helm will stop waiting when 1 of replicas are ready
+    # We need to wait for all replicas to be ready to be able to compute monitorable and monitored pods
+    await wait_for_all_replicaset_replicas_ready(kube_client, generated_data.ess_namespace)
     all_monitorable_pods = set()
     async for pod in kube_client.list(
         Pod, namespace=generated_data.ess_namespace, labels={"app.kubernetes.io/part-of": op.in_(["matrix-stack"])}