fix: prevent watch process crash on invalid template and improve sync test reliability (#8652)

- watch_manager: wrap _add_template_triggers in try/except within the
  infra sync error handler to prevent the watch process from crashing
  when the template is unparseable during error recovery
- testing_utils: add descriptive message to bare ValueError in
  read_until, and catch ValueError in read_until_string to include
  expected output context
- test_sync_adl: increase layer sync timeouts from 60s to 120s
- test_sync_watch: increase nested stack infra sync timeout to 900s

Author: bnusunny

samcli/lib/sync/watch_manager.py

@@ -277,7 +277,13 @@ def _execute_infra_sync(self, first_sync: bool = False) -> None:
             )
             # Unschedule all triggers and only add back the template one as infra sync is incorrect.
             self._observer.unschedule_all()
-            self._add_template_triggers()
+            try:
+                self._add_template_triggers()
+            except Exception as trigger_err:
+                LOG.warning(
+                    "Failed to re-add template triggers, will retry on next change: %s",
+                    trigger_err,
+                )
         else:
             # Update stacks and repopulate triggers
             # Trigger are not removed until infra sync is finished as there

tests/integration/sync/test_sync_adl.py

@@ -129,7 +129,7 @@ def test_sync_watch_code(self):
         read_until_string(
             self.watch_process,
             "Finished syncing Layer HelloWorldFunction",
-            timeout=60,
+            timeout=120,
         )
         lambda_response = json.loads(self._get_lambda_response(lambda_functions[0]))
         self.assertEqual(lambda_response.get("message"), "hello mars")
@@ -143,7 +143,7 @@ def test_sync_watch_code(self):
         read_until_string(
             self.watch_process,
             "Finished syncing Layer HelloWorldFunction",
-            timeout=60,
+            timeout=120,
         )
         self._confirm_lambda_error(lambda_functions[0])
 
@@ -155,7 +155,7 @@ def test_sync_watch_code(self):
         read_until_string(
             self.watch_process,
             "Finished syncing Function Layer Reference Sync HelloWorldFunction.\x1b[0m\n",
-            timeout=60,
+            timeout=120,
         )
 
         def _verify_lambda_response(_lambda_response):

tests/integration/sync/test_sync_watch.py

@@ -301,7 +301,7 @@ def test_sync_watch_infra_nested_stack(self):
             self.test_data_path.joinpath("infra", "template-python-before.yaml"),
         )
 
-        read_until_string(self.watch_process, "\x1b[32mInfra sync completed.\x1b[0m\n", timeout=600)
+        read_until_string(self.watch_process, "\x1b[32mInfra sync completed.\x1b[0m\n", timeout=900)
 
         # Updated Infra Validation
         self.stack_resources = self._get_stacks(self.stack_name)

tests/testing_utils.py

@@ -243,6 +243,11 @@ def _compare_output(output, _: List[str]) -> bool:
         raise TimeoutError(
             f"Did not get expected output after {timeout} seconds. Expected output: {expected_output_bytes!r}"
         ) from ex
+    except ValueError as ex:
+        expected_output_bytes = expected_output.encode("utf-8")
+        raise ValueError(
+            f"Process ended before expected output was found. Expected output: {expected_output_bytes!r}"
+        ) from ex
 
 
 def read_until(process: Popen, callback: Callable[[str, List[str]], bool], timeout: int = 5):
@@ -285,7 +290,7 @@ def _read_output():
         if isinstance(result, Exception):
             raise result
     else:
-        raise ValueError()
+        raise ValueError("Process ended before expected output was found.")
 
 
 class FileCreator(object):