AssertionError: Checksums were not equal. The new checksums have been written to /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/e425dd3ccb181438fb02b529695367d643a18984/checksum/historical-24hr-checksum.json.
assert {'output': {'...ion': '1-0-0'} == {'output': {'...ion': '1-0-0'}
  
  Omitting 1 identical items, use -vv to show
  Differing items:
  {'output': {'Advection of u': ['371595032323388132', '-442653630039549212'], 'Advection of v': ['-8268804406027965017'...ional velocity': ['-7019234032349371747', '5092753012344771320'], 'Thickness%depth_st': ['-2190684862846133381'], ...}} != {'output': {'Advection of u': ['4285061999084278385', '2912608331260814462'], 'Advection of v': ['2169634967076007290'...onal velocity': ['-5170632961632844400', '-1981976763722998053'], 'Thickness%depth_st': ['-2190684862846133371'], ...}}
  Use -v to get more diff
self = <test_bit_reproducibility.TestBitReproducibility object at 0x7f7c6b310f40>
output_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/e425dd3ccb181438fb02b529695367d643a18984')
control_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/e425dd3ccb181438fb02b529695367d643a18984/base-experiment')
experiments = <model_config_tests.exp_test_helper.Experiments object at 0x7f7c6b313e80>
checksum_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/e425dd3ccb181438fb02b529695367d643a18984/compared/testing/checksum/historical-24hr-checksum.json')

    @pytest.mark.repro
    @pytest.mark.repro_historical
    @pytest.mark.experiments(
        {
            EXP_DEFAULT_RUNTIME: {"n_runs": 1},
        }
    )
    def test_repro_historical(
        self,
        output_path: Path,
        control_path: Path,
        experiments: Experiments,
        checksum_path: Optional[Path],
    ):
        """
        Historical reproducibility test that confirms results from a model
        run match a stored previous result. Any generated results are
        added to a "checksum" subdirectory in the output directory.
    
        Parameters
        ----------
        output_path: Path
            Output directory for test output and where the control and
            lab directories are stored for the payu experiments. Default is
            set in conftest.py.
        control_path: Path
            Path to the model configuration to test. This is copied for
            for control directories in experiments. Default is set in
            conftests.py.
        experiments: Experiments
            Class that manages the shared experiments. This is a fixture
            defined in this file.
        checksum_path: Optional[Path]
            Path to checksums to compare model output against. Default is
            set to checksums saved on model configuration. This is a
            fixture defined in conftests.py
        """
        # Get output directory for the checksums
        checksum_output_dir = set_checksum_output_dir(output_path=output_path)
    
        # Use default runtime experiment to get the historical checksums
        experiments.check_experiments([EXP_DEFAULT_RUNTIME])
        exp = experiments.get_experiment(EXP_DEFAULT_RUNTIME)
    
        assert (
            exp.model.output_exists()
        ), "Output file required for model checksums does not exist"
    
        # Set the checksum output filename using the model default runtime
        runtime_hours = exp.model.default_runtime_seconds // HOUR_IN_SECONDS
        checksum_filename = f"historical-{runtime_hours}hr-checksum.json"
    
        # Read the historical checksum file
        hist_checksums = read_historical_checksums(
            control_path, checksum_filename, checksum_path
        )
    
        # Use historical file checksums schema version for parsing checksum,
        # otherwise use the model default, if file does not exist
        schema_version = (
            hist_checksums["schema_version"]
            if hist_checksums
            else exp.model.default_schema_version
        )
    
        # Extract checksums
        checksums = exp.extract_checksums(schema_version=schema_version)
    
        # Write out checksums to output file
        checksum_output_file = checksum_output_dir / checksum_filename
        with open(checksum_output_file, "w") as file:
            json.dump(checksums, file, indent=2)
    
>       assert (
            hist_checksums == checksums
        ), f"Checksums were not equal. The new checksums have been written to {checksum_output_file}."
E       AssertionError: Checksums were not equal. The new checksums have been written to /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/e425dd3ccb181438fb02b529695367d643a18984/checksum/historical-24hr-checksum.json.
E       assert {'output': {'...ion': '1-0-0'} == {'output': {'...ion': '1-0-0'}
E         
E         Omitting 1 identical items, use -vv to show
E         Differing items:
E         {'output': {'Advection of u': ['371595032323388132', '-442653630039549212'], 'Advection of v': ['-8268804406027965017'...ional velocity': ['-7019234032349371747', '5092753012344771320'], 'Thickness%depth_st': ['-2190684862846133381'], ...}} != {'output': {'Advection of u': ['4285061999084278385', '2912608331260814462'], 'Advection of v': ['2169634967076007290'...onal velocity': ['-5170632961632844400', '-1981976763722998053'], 'Thickness%depth_st': ['-2190684862846133371'], ...}}
E         Use -v to get more diff

../test-venv/lib/python3.10/site-packages/model_config_tests/config_tests/test_bit_reproducibility.py:225: AssertionError