failed on setup with "RuntimeError: Failed to run payu setup:
Return code: 1
--- stdout ---
laboratory path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab
binary path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/bin
input path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/input
work path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/work
archive path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/archive
Experiment name is configured in config.yaml:  exp_default_runtime
payu: Found modules in /opt/Modules/v4.3.0
Loading input manifest: manifests/input.yaml
Loading restart manifest: manifests/restart.yaml
Loading exe manifest: manifests/exe.yaml
Setting up atmosphere
Setting up ocean
Setting up ice
Setting up coupler
Checking exe, input and restart manifests
Manifest path: stored hash != calculated hash
  work/atmosphere/um_hg3.exe: md5: 77e66471e73ae272e6b548f2bc5f2794 != d270e54ded451236d4e3cef867cace27
  work/ice/cice_access-esm1.6_360x300_12x1_12p.exe: md5: 1b2bdff3029668fa3249f69207e2d690 != 4c8aa14fe951d42fa793bacd1c0a709c
  work/ocean/mom5_access_cm: md5: 8e404189b0972fb1beb916ed0dcba88c != 710277fe911adb7eea2e6cdd6ff2a373

--- stderr ---
Loading access-esm1p6/pr181-2
  Loading requirement:
    access-esm1p6/dependencies/pr181-2/oasis3-mct/OASIS3-MCT_5.2-sm7ksjm
    access-esm1p6/dependencies/pr181-2/cice5/access-esm1.6-2025.11.001-wfnrfml
    access-esm1p6/dependencies/pr181-2/access-fms/mom5-2025.08.000-ltn3e2q
    access-esm1p6/dependencies/pr181-2/access-mocsy/2025.07.002-ejsylbt
    access-esm1p6/dependencies/pr181-2/access-generic-tracers/2025.09.000-x2czqdx access-esm1p6/dependencies/pr181-2/mom5/2025.05.000-e3fa5ml access-esm1p6/dependencies/pr181-2/cable/2025.11.000-7ey6m4f access-esm1p6/dependencies/pr181-2/gcom4/2025.08.000-qhkz6ut access-esm1p6/dependencies/pr181-2/um7/2025.12.000-yhsp33y
Run cannot reproduce: manifest manifests/exe.yaml is not correct"
request = <SubRequest 'experiments' for <Function test_repro_historical>>
output_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22')
control_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/base-experiment')
keep_archive = False

    @pytest.fixture(scope="class")
    def experiments(
        request, output_path: Path, control_path: Path, keep_archive: Optional[bool]
    ):
        """
        Parse the experiments markers from the requested tests and
        submit all necessary experiments at the same time.
    
        The scope is class so the experiments are only run once before all repro
        tests.
        """
    
        # Parse the experiments markers from the requested tests
        experiments_markers = []
        for item in request.session.items:
            if item.parent == request.node:
                marker = item.get_closest_marker("experiments")
                if marker:
                    experiments_markers.append(marker.args[0])
    
>       return _experiments(experiments_markers, output_path, control_path, keep_archive)

../test-venv/lib/python3.10/site-packages/model_config_tests/config_tests/test_bit_reproducibility.py:147: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../test-venv/lib/python3.10/site-packages/model_config_tests/config_tests/test_bit_reproducibility.py:113: in _experiments
    experiments.setup_and_submit(
../test-venv/lib/python3.10/site-packages/model_config_tests/exp_test_helper.py:279: in setup_and_submit
    exp.submit_payu_run(n_runs=n_runs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <model_config_tests.exp_test_helper.ExpTestHelper object at 0x7faf7b7c4d60>
n_runs = 1

    def submit_payu_run(self, n_runs: int = None) -> str:
        """
        Submit a payu run job.
    
        Parameters
        ----------
        n_runs: int
            The number of runs to submit with --nruns.
    
        Returns
        ----------
        str
            The job ID of the submitted payu run job
        """
        if self.disable_payu_run:
            return
    
        owd = Path.cwd()
        try:
            # Change to experiment directory and run.
            os.chdir(self.control_path)
    
            print("Running payu setup")
            result = sp.run(
                ["payu", "setup", "--lab", str(self.lab_path)],
                capture_output=True,
                text=True,
            )
            if result.returncode != 0:
                # Add additional error messaging for debugging
                error_msg = (
                    "Failed to run payu setup:\n"
                    f"Return code: {result.returncode}\n"
                    f"--- stdout ---\n{result.stdout}\n"
                    f"--- stderr ---\n{result.stderr}"
                )
                print(error_msg)
>               raise RuntimeError(error_msg)
E               RuntimeError: Failed to run payu setup:
E               Return code: 1
E               --- stdout ---
E               laboratory path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab
E               binary path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/bin
E               input path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/input
E               work path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/work
E               archive path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/archive
E               Experiment name is configured in config.yaml:  exp_default_runtime
E               payu: Found modules in /opt/Modules/v4.3.0
E               Loading input manifest: manifests/input.yaml
E               Loading restart manifest: manifests/restart.yaml
E               Loading exe manifest: manifests/exe.yaml
E               Setting up atmosphere
E               Setting up ocean
E               Setting up ice
E               Setting up coupler
E               Checking exe, input and restart manifests
E               Manifest path: stored hash != calculated hash
E                 work/atmosphere/um_hg3.exe: md5: 77e66471e73ae272e6b548f2bc5f2794 != d270e54ded451236d4e3cef867cace27
E                 work/ice/cice_access-esm1.6_360x300_12x1_12p.exe: md5: 1b2bdff3029668fa3249f69207e2d690 != 4c8aa14fe951d42fa793bacd1c0a709c
E                 work/ocean/mom5_access_cm: md5: 8e404189b0972fb1beb916ed0dcba88c != 710277fe911adb7eea2e6cdd6ff2a373
E               
E               --- stderr ---
E               Loading access-esm1p6/pr181-2
E                 Loading requirement:
E                   access-esm1p6/dependencies/pr181-2/oasis3-mct/OASIS3-MCT_5.2-sm7ksjm
E                   access-esm1p6/dependencies/pr181-2/cice5/access-esm1.6-2025.11.001-wfnrfml
E                   access-esm1p6/dependencies/pr181-2/access-fms/mom5-2025.08.000-ltn3e2q
E                   access-esm1p6/dependencies/pr181-2/access-mocsy/2025.07.002-ejsylbt
E                   access-esm1p6/dependencies/pr181-2/access-generic-tracers/2025.09.000-x2czqdx access-esm1p6/dependencies/pr181-2/mom5/2025.05.000-e3fa5ml access-esm1p6/dependencies/pr181-2/cable/2025.11.000-7ey6m4f access-esm1p6/dependencies/pr181-2/gcom4/2025.08.000-qhkz6ut access-esm1p6/dependencies/pr181-2/um7/2025.12.000-yhsp33y
E               Run cannot reproduce: manifest manifests/exe.yaml is not correct

../test-venv/lib/python3.10/site-packages/model_config_tests/exp_test_helper.py:154: RuntimeError

failed on setup with "RuntimeError: Failed to run payu setup:
Return code: 1
--- stdout ---
laboratory path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab
binary path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/bin
input path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/input
work path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/work
archive path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/archive
Experiment name is configured in config.yaml:  exp_default_runtime
payu: Found modules in /opt/Modules/v4.3.0
Loading input manifest: manifests/input.yaml
Loading restart manifest: manifests/restart.yaml
Loading exe manifest: manifests/exe.yaml
Setting up atmosphere
Setting up ocean
Setting up ice
Setting up coupler
Checking exe, input and restart manifests
Manifest path: stored hash != calculated hash
  work/atmosphere/um_hg3.exe: md5: 77e66471e73ae272e6b548f2bc5f2794 != d270e54ded451236d4e3cef867cace27
  work/ice/cice_access-esm1.6_360x300_12x1_12p.exe: md5: 1b2bdff3029668fa3249f69207e2d690 != 4c8aa14fe951d42fa793bacd1c0a709c
  work/ocean/mom5_access_cm: md5: 8e404189b0972fb1beb916ed0dcba88c != 710277fe911adb7eea2e6cdd6ff2a373

--- stderr ---
Loading access-esm1p6/pr181-2
  Loading requirement:
    access-esm1p6/dependencies/pr181-2/oasis3-mct/OASIS3-MCT_5.2-sm7ksjm
    access-esm1p6/dependencies/pr181-2/cice5/access-esm1.6-2025.11.001-wfnrfml
    access-esm1p6/dependencies/pr181-2/access-fms/mom5-2025.08.000-ltn3e2q
    access-esm1p6/dependencies/pr181-2/access-mocsy/2025.07.002-ejsylbt
    access-esm1p6/dependencies/pr181-2/access-generic-tracers/2025.09.000-x2czqdx access-esm1p6/dependencies/pr181-2/mom5/2025.05.000-e3fa5ml access-esm1p6/dependencies/pr181-2/cable/2025.11.000-7ey6m4f access-esm1p6/dependencies/pr181-2/gcom4/2025.08.000-qhkz6ut access-esm1p6/dependencies/pr181-2/um7/2025.12.000-yhsp33y
Run cannot reproduce: manifest manifests/exe.yaml is not correct"
request = <SubRequest 'experiments' for <Function test_repro_historical>>
output_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22')
control_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/base-experiment')
keep_archive = False

    @pytest.fixture(scope="class")
    def experiments(
        request, output_path: Path, control_path: Path, keep_archive: Optional[bool]
    ):
        """
        Parse the experiments markers from the requested tests and
        submit all necessary experiments at the same time.
    
        The scope is class so the experiments are only run once before all repro
        tests.
        """
    
        # Parse the experiments markers from the requested tests
        experiments_markers = []
        for item in request.session.items:
            if item.parent == request.node:
                marker = item.get_closest_marker("experiments")
                if marker:
                    experiments_markers.append(marker.args[0])
    
>       return _experiments(experiments_markers, output_path, control_path, keep_archive)

../test-venv/lib/python3.10/site-packages/model_config_tests/config_tests/test_bit_reproducibility.py:147: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../test-venv/lib/python3.10/site-packages/model_config_tests/config_tests/test_bit_reproducibility.py:113: in _experiments
    experiments.setup_and_submit(
../test-venv/lib/python3.10/site-packages/model_config_tests/exp_test_helper.py:279: in setup_and_submit
    exp.submit_payu_run(n_runs=n_runs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <model_config_tests.exp_test_helper.ExpTestHelper object at 0x7faf7b7c4d60>
n_runs = 1

    def submit_payu_run(self, n_runs: int = None) -> str:
        """
        Submit a payu run job.
    
        Parameters
        ----------
        n_runs: int
            The number of runs to submit with --nruns.
    
        Returns
        ----------
        str
            The job ID of the submitted payu run job
        """
        if self.disable_payu_run:
            return
    
        owd = Path.cwd()
        try:
            # Change to experiment directory and run.
            os.chdir(self.control_path)
    
            print("Running payu setup")
            result = sp.run(
                ["payu", "setup", "--lab", str(self.lab_path)],
                capture_output=True,
                text=True,
            )
            if result.returncode != 0:
                # Add additional error messaging for debugging
                error_msg = (
                    "Failed to run payu setup:\n"
                    f"Return code: {result.returncode}\n"
                    f"--- stdout ---\n{result.stdout}\n"
                    f"--- stderr ---\n{result.stderr}"
                )
                print(error_msg)
>               raise RuntimeError(error_msg)
E               RuntimeError: Failed to run payu setup:
E               Return code: 1
E               --- stdout ---
E               laboratory path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab
E               binary path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/bin
E               input path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/input
E               work path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/work
E               archive path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/595754a108976da9395522516b4066ab0ccf3a22/lab/archive
E               Experiment name is configured in config.yaml:  exp_default_runtime
E               payu: Found modules in /opt/Modules/v4.3.0
E               Loading input manifest: manifests/input.yaml
E               Loading restart manifest: manifests/restart.yaml
E               Loading exe manifest: manifests/exe.yaml
E               Setting up atmosphere
E               Setting up ocean
E               Setting up ice
E               Setting up coupler
E               Checking exe, input and restart manifests
E               Manifest path: stored hash != calculated hash
E                 work/atmosphere/um_hg3.exe: md5: 77e66471e73ae272e6b548f2bc5f2794 != d270e54ded451236d4e3cef867cace27
E                 work/ice/cice_access-esm1.6_360x300_12x1_12p.exe: md5: 1b2bdff3029668fa3249f69207e2d690 != 4c8aa14fe951d42fa793bacd1c0a709c
E                 work/ocean/mom5_access_cm: md5: 8e404189b0972fb1beb916ed0dcba88c != 710277fe911adb7eea2e6cdd6ff2a373
E               
E               --- stderr ---
E               Loading access-esm1p6/pr181-2
E                 Loading requirement:
E                   access-esm1p6/dependencies/pr181-2/oasis3-mct/OASIS3-MCT_5.2-sm7ksjm
E                   access-esm1p6/dependencies/pr181-2/cice5/access-esm1.6-2025.11.001-wfnrfml
E                   access-esm1p6/dependencies/pr181-2/access-fms/mom5-2025.08.000-ltn3e2q
E                   access-esm1p6/dependencies/pr181-2/access-mocsy/2025.07.002-ejsylbt
E                   access-esm1p6/dependencies/pr181-2/access-generic-tracers/2025.09.000-x2czqdx access-esm1p6/dependencies/pr181-2/mom5/2025.05.000-e3fa5ml access-esm1p6/dependencies/pr181-2/cable/2025.11.000-7ey6m4f access-esm1p6/dependencies/pr181-2/gcom4/2025.08.000-qhkz6ut access-esm1p6/dependencies/pr181-2/um7/2025.12.000-yhsp33y
E               Run cannot reproduce: manifest manifests/exe.yaml is not correct

../test-venv/lib/python3.10/site-packages/model_config_tests/exp_test_helper.py:154: RuntimeError