Merge pull request #154 from ACCESS-NRI/148-compare-multi-experiments-repro-command

Add multi experiment repro test command:  `compare-exp-tests`

Given a list of experiment directories with the `--dirs` flag, generates tests that compares the checksums of the first output between each pair of experiments.

Author: jo-basevi

README.md

@@ -6,7 +6,7 @@ The checksum pytests are used for reproducibility CI checks in this repository.
 
 Code from these pytests is adapted from COSIMAS's ACCESS-OM2's [bit reproducibility tests](https://github.com/COSIMA/access-om2/blob/master/test/test_bit_reproducibility.py).
 
-## Pytests
+## model-config-tests Pytests
 
 ### How to run pytests manually on NCI
 
@@ -127,6 +127,21 @@ It is helpful if we work from the same definitions of what 'reproducibility' mea
 
 Code block names above indicate tests that are currently available using the CI/CD system, see [pytest markers](#pytest_markers)
 
+## Check for pairwise reproducibility between multiple experiments
+
+To compare output/checksums between pairs of
+experiment directories, use the `compare-exp-tests` command. For example
+
+```
+compare-exp-tests --dirs "path/to/exp1 path/to/exp2 path/to/exp3"
+```
+
+The `--dirs` option specifies a space separated list of experiment directories to compare. These can be relative or absolute paths and should point to payu control directories. The above example generates three pairwise tests: "exp1 vs exp2", "exp1 vs exp3", and "exp2 vs exp3".
+
+To enable more detailed output during the test runs, add the `-vvv` flag to the command.
+
+Currently these tests only compare the first model run output directory.
+
 ## CI/CD
 
 The `.github` directory contains many different workflows and actions. This section describes how they are used.

pyproject.toml

@@ -21,7 +21,8 @@ dependencies = [
     "pytest >=8.0.1",
     "ruamel.yaml >=0.18.5",
     "jsonschema >=4.21.1",
-    "payu >=1.1.3"
+    "payu >=1.1.3",
+    "pytest-sugar"
 ]
 
 [project.optional-dependencies]
@@ -30,7 +31,8 @@ test = [
 ]
 
 [project.scripts]
-model-config-tests = "model_config_tests.__main__:main"
+model-config-tests = "model_config_tests.cmds.config_tests_cmd:main"
+compare-exp-tests = "model_config_tests.cmds.compare_exp_tests_cmd:main"
 
 [project.urls]
 Homepage = "https://github.com/ACCESS-NRI/model-config-tests/"

src/model_config_tests/cmds/compare_exp_tests_cmd.py

@@ -0,0 +1,25 @@
+"""Tests for comparing multiple experiments results"""
+
+import sys
+from pathlib import Path
+
+# Running pytests using --pyargs does not run pytest_addoption in conftest.py
+# Using workaround as described here:
+# https://stackoverflow.com/questions/41270604/using-command-line-parameters-with-pytest-pyargs
+HERE = Path(__file__)
+COMPARE_EXP_TESTS_DIR = "compare_exp_tests"
+
+
+def main():
+    import pytest
+
+    test_path = str(HERE.parent.parent / COMPARE_EXP_TESTS_DIR)
+
+    # Specify --rootdir to shorten relative paths in displayed test output
+    errcode = pytest.main([test_path] + sys.argv[1:] + [f"--rootdir={test_path}"])
+
+    sys.exit(errcode)
+
+
+if __name__ == "__main__":
+    main()

src/model_config_tests/__main__.py …el_config_tests/cmds/config_tests_cmd.pysrc/model_config_tests/__main__.py renamed to src/model_config_tests/cmds/config_tests_cmd.py src/model_config_tests/__main__.py renamed to src/model_config_tests/cmds/config_tests_cmd.py

@@ -1,16 +1,21 @@
-import os
+"""Tests for a singular model configuration"""
+
 import sys
+from pathlib import Path
 
 # Running pytests using --pyargs does not run pytest_addoption in conftest.py
 # Using workaround as described here:
 # https://stackoverflow.com/questions/41270604/using-command-line-parameters-with-pytest-pyargs
-HERE = os.path.dirname(__file__)
+HERE = Path(__file__)
+CONFIG_TESTS_DIR = "config_tests"
 
 
 def main():
     import pytest
 
-    errcode = pytest.main([HERE] + sys.argv[1:])
+    test_path = str(HERE.parent.parent / CONFIG_TESTS_DIR)
+
+    errcode = pytest.main([test_path] + sys.argv[1:])
     sys.exit(errcode)
 
 

src/model_config_tests/compare_exp_tests/conftest.py

@@ -0,0 +1,71 @@
+# Copyright 2024 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details.
+# SPDX-License-Identifier: Apache-2.0
+from itertools import combinations
+from pathlib import Path
+
+
+# Set up command line options and default for directory paths
+def pytest_addoption(parser):
+    """Attaches custom command line arguments"""
+    parser.addoption(
+        "--dirs",
+        action="store",
+        help="Specify a space separated list of experiment control directories to compare",
+    )
+
+
+def pytest_generate_tests(metafunc):
+    """Set up dynamic parametrisation for testing pairwise comparisons
+    of input directories"""
+    if (
+        "experiment_1" in metafunc.fixturenames
+        and "experiment_2" in metafunc.fixturenames
+    ):
+        # Generate pairs of experiments from command input
+        input_dirs = metafunc.config.getoption("dirs")
+        dir_pairs = get_experiment_pairs(input_dirs)
+
+        # Generate some readable IDs for the pairs
+        ids = [f"{exp1.name} vs {exp2.name}" for exp1, exp2 in dir_pairs]
+        metafunc.parametrize("experiment_1,experiment_2", dir_pairs, ids=ids)
+
+
+def get_experiment_pairs(dirs):
+    """Generate experiment directory pairs
+
+    Parameters
+    ----------
+    dirs : str
+        Space separated list of directories to compare
+
+    Returns
+    -------
+    list[tuple[Path, Path]]
+        List of pairs of directories to compare
+    """
+    if dirs is None:
+        raise ValueError(
+            "No directories specified, use --dirs to specify a space separated list"
+        )
+
+    dirs = dirs.split()
+
+    paths = set()
+    for dir in dirs:
+        # Check if the path exists and is a directory
+        path = Path(dir)
+        if not path.exists():
+            raise ValueError(f"Directory {dir} does not exist")
+        if not path.is_dir():
+            raise ValueError(f"Path {dir} is not a directory")
+
+        # Resolve to absolute path
+        path = path.resolve()
+        paths.add(path)
+
+    if len(paths) < 2:
+        raise ValueError("Need at least two directories with --dirs to compare")
+
+    paths = sorted(list(paths))
+    dir_pairs = list(combinations(paths, 2))
+    return dir_pairs

src/model_config_tests/compare_exp_tests/test_repro.py

@@ -0,0 +1,43 @@
+# Copyright 2024 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details.
+# SPDX-License-Identifier: Apache-2.0
+from pathlib import Path
+
+from model_config_tests.exp_test_helper import ExpTestHelper
+
+
+def get_lab_path(experiment: Path) -> Path:
+    """
+    Derive the lab path from the experiment configuration directory
+    archive symlink
+    """
+    archive_symlink = experiment / "archive"
+    if not archive_symlink.is_symlink():
+        raise ValueError(f"Archive symlink does not exist in {experiment}.")
+
+    # experiment/archive symlink points to lab_path/archive/exp_name
+    return archive_symlink.resolve().parent.parent
+
+
+def test_pairwise_repro(experiment_1: Path, experiment_2: Path):
+    """
+    Compare combinations of experiments to check for reproducibility.
+    This is parametrised in conftest with pytest_generate_tests to
+    dynamically generate pairs of experiments to compare.
+    """
+
+    lab_path1 = get_lab_path(experiment_1)
+    exp1 = ExpTestHelper(
+        control_path=experiment_1, lab_path=lab_path1, disable_payu_run=True
+    )
+
+    lab_path2 = get_lab_path(experiment_2)
+    exp2 = ExpTestHelper(
+        control_path=experiment_2, lab_path=lab_path2, disable_payu_run=True
+    )
+
+    # Compare the two experiments - compares checksums from output000
+    exp1_checksums = exp1.model.extract_full_checksums()
+    exp2_checksums = exp2.model.extract_full_checksums()
+    assert (
+        exp1_checksums == exp2_checksums
+    ), f"Checksums do not match for {experiment_1.name} and {experiment_2.name} experiments"

src/model_config_tests/conftest.py …el_config_tests/config_tests/conftest.pysrc/model_config_tests/conftest.py renamed to src/model_config_tests/config_tests/conftest.py src/model_config_tests/conftest.py renamed to src/model_config_tests/config_tests/conftest.py

@@ -10,7 +10,9 @@
 import f90nml
 import pytest
 
-from model_config_tests.qa.test_config import check_manifest_exes_in_spack_location
+from model_config_tests.config_tests.qa.test_config import (
+    check_manifest_exes_in_spack_location,
+)
 from model_config_tests.util import get_git_branch_name
 
 # Name of module on NCI

…ig_tests/qa/test_access_esm1p5_config.py …ig_tests/qa/test_access_esm1p5_config.pysrc/model_config_tests/qa/test_access_esm1p5_config.py renamed to src/model_config_tests/config_tests/qa/test_access_esm1p5_config.py src/model_config_tests/qa/test_access_esm1p5_config.py renamed to src/model_config_tests/config_tests/qa/test_access_esm1p5_config.py

@@ -10,7 +10,9 @@
 import f90nml
 import pytest
 
-from model_config_tests.qa.test_config import check_manifest_exes_in_spack_location
+from model_config_tests.config_tests.qa.test_config import (
+    check_manifest_exes_in_spack_location,
+)
 from model_config_tests.util import get_git_branch_name
 
 # Name of module on NCI

…ig_tests/qa/test_access_esm1p6_config.py …ig_tests/qa/test_access_esm1p6_config.pysrc/model_config_tests/qa/test_access_esm1p6_config.py renamed to src/model_config_tests/config_tests/qa/test_access_esm1p6_config.py src/model_config_tests/qa/test_access_esm1p6_config.py renamed to src/model_config_tests/config_tests/qa/test_access_esm1p6_config.py

@@ -9,7 +9,9 @@
 import f90nml
 import pytest
 
-from model_config_tests.qa.test_config import check_manifest_exes_in_spack_location
+from model_config_tests.config_tests.qa.test_config import (
+    check_manifest_exes_in_spack_location,
+)
 from model_config_tests.util import get_git_branch_name
 
 # Mutually exclusive topic keywords