AssertionError: There was an error running experiment: exp_default_runtime
self = <test_bit_reproducibility.TestBitReproducibility object at 0x7fee29857ac0>
output_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/ae6c541afc9cb2674f756aa20f1d3a6bf1a33005')
control_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/ae6c541afc9cb2674f756aa20f1d3a6bf1a33005/base-experiment')
experiments = <model_config_tests.exp_test_helper.Experiments object at 0x7fee2a182f50>
checksum_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/ae6c541afc9cb2674f756aa20f1d3a6bf1a33005/compared/testing/checksum/historical-24hr-checksum.json')

    @pytest.mark.repro
    @pytest.mark.repro_historical
    @pytest.mark.experiments(
        {
            EXP_DEFAULT_RUNTIME: {"n_runs": 1},
        }
    )
    def test_repro_historical(
        self,
        output_path: Path,
        control_path: Path,
        experiments: Experiments,
        checksum_path: Optional[Path],
    ):
        """
        Historical reproducibility test that confirms results from a model
        run match a stored previous result. Any generated results are
        added to a "checksum" subdirectory in the output directory.
    
        Parameters
        ----------
        output_path: Path
            Output directory for test output and where the control and
            lab directories are stored for the payu experiments. Default is
            set in conftest.py.
        control_path: Path
            Path to the model configuration to test. This is copied for
            for control directories in experiments. Default is set in
            conftests.py.
        experiments: Experiments
            Class that manages the shared experiments. This is a fixture
            defined in this file.
        checksum_path: Optional[Path]
            Path to checksums to compare model output against. Default is
            set to checksums saved on model configuration. This is a
            fixture defined in conftests.py
        """
        # Get output directory for the checksums
        checksum_output_dir = set_checksum_output_dir(output_path=output_path)
    
        # Use default runtime experiment to get the historical checksums
>       experiments.check_experiments([EXP_DEFAULT_RUNTIME])

../test-venv/lib/python3.10/site-packages/model_config_tests/config_tests/test_bit_reproducibility.py:193: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <model_config_tests.exp_test_helper.Experiments object at 0x7fee2a182f50>
exp_names = ['exp_default_runtime']

    def check_experiments(self, exp_names=list[str]) -> None:
        """
        Check whether given experiments names have run successfully
        """
        for exp_name in exp_names:
            # TODO: Is there other useful information to display here?
>           assert (
                exp_name in self.successful_experiments
            ), f"There was an error running experiment: {exp_name}"
E           AssertionError: There was an error running experiment: exp_default_runtime

../test-venv/lib/python3.10/site-packages/model_config_tests/exp_test_helper.py:298: AssertionError

AssertionError: There was an error running experiment: exp_1d_runtime
self = <test_bit_reproducibility.TestBitReproducibility object at 0x7fee29857e20>
experiments = <model_config_tests.exp_test_helper.Experiments object at 0x7fee2a182f50>

    @pytest.mark.repro
    @pytest.mark.repro_determinism
    @pytest.mark.slow
    @pytest.mark.experiments(
        {
            EXP_1D_RUNTIME: {"n_runs": 1, "model_runtime": DAY_IN_SECONDS},
            EXP_1D_RUNTIME_REPEAT: {"n_runs": 1, "model_runtime": DAY_IN_SECONDS},
        }
    )
    def test_repro_determinism(self, experiments: Experiments):
        """
        Determinism test that confirms repeated model runs for 1 day
        give the same results
        """
>       experiments.check_experiments([EXP_1D_RUNTIME, EXP_1D_RUNTIME_REPEAT])

../test-venv/lib/python3.10/site-packages/model_config_tests/config_tests/test_bit_reproducibility.py:243: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <model_config_tests.exp_test_helper.Experiments object at 0x7fee2a182f50>
exp_names = ['exp_1d_runtime', 'exp_1d_runtime_repeat']

    def check_experiments(self, exp_names=list[str]) -> None:
        """
        Check whether given experiments names have run successfully
        """
        for exp_name in exp_names:
            # TODO: Is there other useful information to display here?
>           assert (
                exp_name in self.successful_experiments
            ), f"There was an error running experiment: {exp_name}"
E           AssertionError: There was an error running experiment: exp_1d_runtime

../test-venv/lib/python3.10/site-packages/model_config_tests/exp_test_helper.py:298: AssertionError