fix: remove fallback in count_running_containers that causes flaky warm container tests

* Update notify-slack.yml to add sleep (aws#8646)

* fix: use tag prefix matching to clean up samcli/lambda-* images (aws#8647)

Docker's images.list(name='samcli/lambda') does exact repository matching
and won't match repositories like 'samcli/lambda-python'. This caused
stale images to persist across parameterized test classes, leading to
flaky test_download_two_layers failures where Layer2 should overwrite
Layer1 but the image was never rebuilt.

Fix by:
1. Adding _cleanup_samcli_images() that lists all images and filters by
   'samcli/lambda-' tag prefix
2. Using this method in both tearDown and tearDownClass
3. Fixing the same pattern in TestLayerVersionThatDoNotCreateCache

* fix: remove fallback in count_running_containers that causes flaky warm container tests

The count_running_containers method had a fallback that returned the count
of ALL SAM CLI containers when MODE env var filtering found no matches.
This caused AssertionError: 3 != 2 when stale containers from other tests
were present. Now it strictly counts only containers matching this test's
unique MODE UUID and uses exact string matching.

* fix: add AWS_DEFAULT_REGION and use -k pattern for parameterized tests

* fix: add BY_CANARY=true to enable Docker tests on CI

* fix: exclude RemoteLayers tests that need AWS credentials

---------

Co-authored-by: seshubaws <116689586+seshubaws@users.noreply.github.com>

Author: bnusunny

.github/workflows/notify-slack.yml

@@ -12,8 +12,11 @@ jobs:
     permissions:
       contents: read
     steps:
+      - name: Wait for label to be applied
+        run: sleep 10s
+        shell: bash
       - name: Send External PR Notification
-        if: github.event_name == 'pull_request' && github.event.label.name == 'pr/external'
+        if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'pr/external')
         uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # v2
         env:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/test-warm-containers-fix.yml

@@ -0,0 +1,44 @@
+name: Test Warm Containers Fix
+
+on:
+  pull_request:
+    branches: [develop]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  AWS_DEFAULT_REGION: us-east-1
+  SAM_CLI_DEV: 1
+  SAM_CLI_TELEMETRY: 0
+  SAM_CLI_CONTAINER_CONNECTION_TIMEOUT: 60
+  BY_CANARY: true
+
+jobs:
+  test-warm-containers:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install SAM CLI in dev mode
+        run: pip install -e ".[dev]"
+
+      - name: Run warm containers tests
+        run: |
+          pytest -vv --reruns 3 \
+            -k "TestWarmContainers and not RemoteLayers" \
+            tests/integration/local/start_api/test_start_api.py \
+            tests/integration/local/start_lambda/test_start_lambda.py \
+            --json-report --json-report-file=test-report.json
+
+      - name: Upload test report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-report
+          path: test-report.json

tests/integration/local/invoke/test_integrations_cli.py

@@ -803,13 +803,27 @@ class TestLayerVersionBase(InvokeIntegBase):
     def setUp(self):
         self.layer_cache = Path().home().joinpath("integ_layer_cache")
 
+    @staticmethod
+    def _cleanup_samcli_images(docker_client):
+        """Remove all samcli/lambda-* images.
+
+        Docker's images.list(name="samcli/lambda") does exact repository matching
+        and won't match repositories like "samcli/lambda-python". We list all images
+        and filter by tag prefix instead.
+        """
+        try:
+            all_images = docker_client.images.list()
+            for image in all_images:
+                for tag in image.tags:
+                    if tag.startswith("samcli/lambda-"):
+                        docker_client.remove_image_safely(image.id, force=True)
+                        break
+        except Exception:
+            pass
+
     def tearDown(self):
         docker_client = get_validated_container_client()
-        samcli_images = docker_client.images.list(name="samcli/lambda")
-        for image in samcli_images:
-            # Use strategy pattern method for runtime-aware image cleanup
-            docker_client.remove_image_safely(image.id, force=True)
-
+        self._cleanup_samcli_images(docker_client)
         shutil.rmtree(str(self.layer_cache), ignore_errors=True)
 
     @classmethod
@@ -823,10 +837,7 @@ def tearDownClass(cls):
         cls.layer_utils.delete_layers()
         # Added to handle the case where ^C failed the test due to invalid cleanup of layers
         docker_client = get_validated_container_client()
-        samcli_images = docker_client.images.list(name="samcli/lambda")
-        for image in samcli_images:
-            # Use strategy pattern method for runtime-aware image cleanup
-            docker_client.remove_image_safely(image.id, force=True)
+        cls._cleanup_samcli_images(docker_client)
         integ_layer_cache_dir = Path().home().joinpath("integ_layer_cache")
         if integ_layer_cache_dir.exists():
             shutil.rmtree(str(integ_layer_cache_dir))
@@ -1069,12 +1080,18 @@ def setUp(self):
     def tearDown(self):
         docker_client = get_validated_container_client()
 
-        # Use strategy pattern method for runtime-aware image cleanup
-        # This handles both Docker and Finch cleanup strategies automatically
-        samcli_images = docker_client.images.list(name="samcli/lambda")
-        for image in samcli_images:
-            # Use strategy pattern method that handles runtime-specific cleanup logic
-            docker_client.remove_image_safely(image.id, force=True)
+        # Use the same prefix-based cleanup as TestLayerVersionBase.
+        # Docker's images.list(name="samcli/lambda") does exact repository matching
+        # and won't match "samcli/lambda-python" etc.
+        try:
+            all_images = docker_client.images.list()
+            for image in all_images:
+                for tag in image.tags:
+                    if tag.startswith("samcli/lambda-"):
+                        docker_client.remove_image_safely(image.id, force=True)
+                        break
+        except Exception:
+            pass
 
     def test_layer_does_not_exist(self):
         self.layer_utils.upsert_layer(LayerUtils.generate_layer_name(), "LayerOneArn", "layer1.zip")

tests/integration/local/start_api/test_start_api.py

@@ -2220,35 +2220,26 @@ def setUp(self):
 
     def count_running_containers(self):
         """Count containers created by this test using Docker client directly."""
-        # Use Docker client to find containers with SAM CLI labels
         try:
-            # Get running containers with SAM CLI lambda container label
             sam_containers = self.docker_client.containers.list(
                 all=False, filters={"label": "sam.cli.container.type=lambda"}
             )
 
-            # Filter by our test's mode environment variable if possible
             test_containers = []
             for container in sam_containers:
                 try:
                     container.reload()
                     env_vars = container.attrs.get("Config", {}).get("Env", [])
                     for env_var in env_vars:
-                        if env_var.startswith("MODE=") and self.mode_env_variable in env_var:
+                        if env_var == f"MODE={self.mode_env_variable}":
                             test_containers.append(container)
                             break
                 except Exception:
                     continue
 
-            # If we found containers with our mode variable, return that count
-            if test_containers:
-                return len(test_containers)
+            return len(test_containers)
 
-            # Otherwise, return all SAM containers (fallback)
-            return len(sam_containers)
-
-        except Exception as e:
-            # If we can't access Docker client, fall back to 0
+        except Exception:
             return 0
 
     def _parse_container_ids_from_output(self):

tests/integration/local/start_lambda/test_start_lambda.py

@@ -403,35 +403,26 @@ def setUp(self):
 
     def count_running_containers(self):
         """Count containers created by this test using Docker client directly."""
-        # Use Docker client to find containers with SAM CLI labels
         try:
-            # Get running containers with SAM CLI lambda container label
             sam_containers = self.docker_client.containers.list(
                 all=False, filters={"label": "sam.cli.container.type=lambda"}
             )
 
-            # Filter by our test's mode environment variable if possible
             test_containers = []
             for container in sam_containers:
                 try:
                     container.reload()
                     env_vars = container.attrs.get("Config", {}).get("Env", [])
                     for env_var in env_vars:
-                        if env_var.startswith("MODE=") and self.mode_env_variable in env_var:
+                        if env_var == f"MODE={self.mode_env_variable}":
                             test_containers.append(container)
                             break
                 except Exception:
                     continue
 
-            # If we found containers with our mode variable, return that count
-            if test_containers:
-                return len(test_containers)
+            return len(test_containers)
 
-            # Otherwise, return all SAM containers (fallback)
-            return len(sam_containers)
-
-        except Exception as e:
-            # If we can't access Docker client, fall back to 0
+        except Exception:
             return 0
 
     def _parse_container_ids_from_output(self):