Add repro payu setup check - this should pick up changes to manifests

README.md

@@ -90,10 +90,15 @@ Running all tests in the pytest suite on a configuration will likely fail as the
 - `repro_determinism`: Determinism test that confirms repeated model runs give the same result.
 - `repro_determinism_restart`: Determinism test that confirms repeated experiments with two consecutive runs give the same result.
 - `repro_restart`: Restart reproducibility test that confirms two short consecutive model runs give the same result as a longer single model run.
+- `repro_payu_setup`: Test payu setup reproducibility; fail if MD5 of any file in manifest is changed.
+- `manifests_unchanged`: Uses `git diff` to check for any unauthorized changes in manifest.
+- `manifests`: A shortcut to run both `manifests_unchanged` and `repro_payu_setup`.
 - `slow`: Tests that are slow to run
 - `dev_config`: General configuration QA tests.
 - `config`: Configuration QA tests for released branches. This includes the `dev_config` tests.
 
+
+
 There are also model-specific markers for configuration QA tests, e.g., `access_om2`, `access_esm1p5`, `access_om3` and `access_esm1p6`. For a list of all available markers,
 run:
 

src/model_config_tests/config_tests/conftest.py

@@ -131,6 +131,18 @@ def pytest_configure(config):
         "markers",
         "repro_determinism_restart: mark tests that check determinism restart",
     )
+    config.addinivalue_line(
+        "markers",
+        "repro_payu_setup: mark tests that check payu setup reproducibility",
+    )
+    config.addinivalue_line(
+        "markers",
+        "manifests: mark tests that check payu setup does not change manifests files or md5",
+    )
+    config.addinivalue_line(
+        "markers",
+        "manifests_unchanged: mark tests that check payu setup does not change manifests files",
+    )
     config.addinivalue_line("markers", "slow: mark tests that are slow to run")
     config.addinivalue_line(
         "markers",

src/model_config_tests/config_tests/test_bit_reproducibility.py

@@ -9,7 +9,7 @@
 
 import pytest
 
-from model_config_tests.exp_test_helper import Experiments, ExpTestHelper
+from model_config_tests.exp_test_helper import Experiments, ExpTestHelper, setup_exp
 from model_config_tests.util import DAY_IN_SECONDS, HOUR_IN_SECONDS
 
 # Names of shared experiments
@@ -328,3 +328,32 @@ def test_repro_determinism_restart(
         )
 
         assert produced == expected
+
+
+@pytest.mark.repro
+@pytest.mark.manifests
+@pytest.mark.repro_payu_setup
+def test_payu_setup_repro_flag(control_path, output_path):
+    """
+    Test payu setup with `--repro` flag which errors if md5 of any files in payu manifests are changed.
+    """
+    experiment = setup_exp(control_path, output_path, exp_name="repro_payu_setup")
+    try:
+        experiment.setup_reproduce()
+    except Exception as error:
+        pytest.fail(f"{error}")
+
+
+@pytest.mark.manifests
+@pytest.mark.manifests_unchanged
+def test_manifests_unchanged(control_path, output_path):
+    """
+    Test payu setup with `git diff` which errors if any files in payu manifests are changed.
+    """
+    experiment = setup_exp(
+        control_path, output_path, exp_name="setup_unchanged_manifests"
+    )
+    try:
+        experiment.setup_manifests_unchanged()
+    except Exception as error:
+        pytest.fail(f"{error}")

src/model_config_tests/exp_test_helper.py

@@ -82,6 +82,87 @@ def has_run(self):
         """
         return self.model.output_exists()
 
+    def setup(self, reproduce=False):
+        """
+        Run payu setup command. If reproduce is True, run with --reproduce flag
+        to check if md5 hashes have changed in the manifests.
+        """
+        owd = Path.cwd()
+        # Change to experiment directory and run.
+        os.chdir(self.control_path)
+
+        try:
+            setup_command = [
+                "payu",
+                "setup",
+                "--lab",
+                str(self.lab_path),
+            ]
+            if reproduce:
+                setup_command.append("--reproduce")
+            print(f"Running payu setup command: {setup_command}")
+            result = sp.run(setup_command, capture_output=True, text=True)
+
+        finally:
+            # Change back to original working directory
+            os.chdir(owd)
+
+        if result.returncode != 0:
+            raise RuntimeError(
+                "Failed to run payu setup"
+                + (" with --reproduce" if reproduce else "")
+                + f"{'='*10}STDOUT{'='*10}\n {result.stdout}\n"
+                f"{'='*10}STDERR{'='*10}\n {result.stderr}\n"
+            )
+
+    def setup_reproduce(self):
+        """
+        Run payu setup with `--repro` flag to check if md5 hashes have changed in the manifests.
+        """
+        self.setup(reproduce=True)
+
+    def setup_manifests_unchanged(self):
+        """
+        Run payu setup command and check if manifests files have been changed with `git diff`.
+        """
+        self.setup(reproduce=False)
+
+        result = sp.run(
+            ["git", "diff", "--name-only", str(self.control_path / "manifests/")],
+            capture_output=True,
+            text=True,
+        )
+
+        if result.returncode != 0:
+            raise RuntimeError(
+                f"Git command failed with exit code {result.returncode}.\n"
+                f"{'='*10}STDOUT{'='*10}\n {result.stdout}\n"
+                f"{'='*10}STDERR{'='*10}\n {result.stderr}\n"
+            )
+        elif result.stdout != "":
+            # Collect and display the top 10 lines of the diff for each modified file
+            files = result.stdout.strip().split("\n")
+            error_message = "Modifications are detected in file:\n"
+            error_message += "\n".join(" - " + file for file in files) + "\n"
+            error_message += "\nIf md5 hashes have changed, this indicates file contents being different."
+            error_message += """
+If binhashes/paths have changed but md5's are the same,
+this will mean the configuration can reproduce the manifests
+but `payu setup` will take longer to run as it needs to re-calculate all the md5 hashes.
+            """
+            for file in files:
+                diff_details = sp.run(
+                    ["git", "-C", str(self.control_path), "diff", f"{file}"],
+                    capture_output=True,
+                    text=True,
+                )
+                diff_lines = diff_details.stdout.splitlines()
+                top_lines = "\n".join(diff_lines[2:12])
+                if len(diff_lines) > 12:
+                    top_lines += "\n... (truncated)"
+                error_message += f"\n{'='*10} Diff for {file} {'='*10}\n{top_lines}\n"
+            raise RuntimeError(f"{error_message}")
+
     def setup_for_test_run(self):
         """
         Various config.yaml settings need to be modified in order to run in the

tests/test_exp_test_helper.py

@@ -1,7 +1,7 @@
 import shutil
 import subprocess
 from pathlib import Path
-from unittest.mock import Mock, patch
+from unittest.mock import MagicMock, Mock, patch
 
 import pytest
 import yaml
@@ -556,3 +556,83 @@ def test_experiments_check_experiment_error(tmp_path):
     )
     with pytest.raises(RuntimeError, match=error_msg):
         exps.check_experiment("error_exp")
+
+
+@patch("subprocess.run")
+def test_setup_reproduce_error(mock_run, exp):
+    """Test that payu setup --repro fails raises an error and return to original work directory"""
+    # Mock the payu setup --repro to fail
+    mock_result = MagicMock()
+    mock_result.returncode = 1
+    mock_result.stderr = "MD5 mismatch"
+    mock_result.stdout = "Check manifest"
+    mock_run.return_value = mock_result
+
+    with pytest.raises(RuntimeError) as excinfo:
+        exp.setup_reproduce()
+
+        assert (
+            f"Failed to run payu setup with --reproduce. Error: {mock_result.stderr}"
+            in str(excinfo.value)
+        )
+        assert f"Full output: {mock_result.stdout}" in str(excinfo.value)
+
+        # assert returning to the original work directory
+        assert Path.cwd() == exp.control_path
+
+
+@patch("subprocess.run")
+def test_setup_manifests_unchanged_fail_setup(mock_run, exp):
+    """Test that an error is raised when payu setup fails in setup_manifests_unchanged()"""
+    # Mock the payu setup --repro to fail with unchanged manifests
+    mock_result = MagicMock()
+    mock_result.returncode = 1
+    mock_result.stderr = "Setup failed"
+    mock_result.stdout = "Payu setup output"
+    mock_run.return_value = mock_result
+
+    # Store original current working directory
+    owd = Path.cwd()
+
+    with pytest.raises(RuntimeError) as excinfo:
+        exp.setup_manifests_unchanged()
+
+    assert "Failed to run payu setup" in str(excinfo.value)
+    assert f"{'='*10}STDOUT{'='*10}\n {mock_result.stdout}\n" in str(excinfo.value)
+
+    # assert returning to the original work directory
+    assert Path.cwd() == owd
+
+
+@patch("subprocess.run")
+def test_setup_manifests_unchanged_show_changes(mock_run, exp):
+    """Test that when manifests are changed, the `git diff` results are printed to stdout"""
+    # Mock the `payu setup` succeed first
+    setup_success = MagicMock(returncode=0, stdout="Payu setup succeeded")
+
+    top_lines = "+new line\n-old line"
+    diff_file = "manifests/input.yaml"
+    # Then mock the `git diff --name-only` to show which files are changed
+    git_diff_name_only = MagicMock(returncode=0, stdout=diff_file)
+
+    # Mock the `git diff` to show the detailed changes in the file
+    git_diff_run = MagicMock(
+        returncode=0, stdout=(f"--- a/{diff_file}\n+++ b/{diff_file}\n" + top_lines)
+    )
+
+    # Run these mocks in sequence
+    mock_run.side_effect = [setup_success, git_diff_name_only, git_diff_run]
+
+    # Store original current working directory
+    owd = Path.cwd()
+
+    with pytest.raises(RuntimeError) as excinfo:
+        exp.setup_manifests_unchanged()
+
+    assert "Modifications are detected in file:\n" in str(excinfo.value)
+    assert f"\n{'='*10} Diff for {diff_file} {'='*10}\n{top_lines}\n" in str(
+        excinfo.value
+    )
+
+    # assert returning to the original work directory
+    assert Path.cwd() == owd