diff --git a/.github/actions/get-deploy-paths/README.md b/.github/actions/get-deploy-paths/README.md index 071b81b7..882f103d 100644 --- a/.github/actions/get-deploy-paths/README.md +++ b/.github/actions/get-deploy-paths/README.md @@ -8,7 +8,8 @@ This action constructs paths relevant to a deployment of `spack`. | ---- | ---- | ----------- | -------- | ------- | ------- | | `spack-installs-root-path` | `string` | Path to a directory within which all versions of spack are installed | `true` | N/A | `"/some/dir/apps/spack"` | | `spack-version` | `string` | Version of spack deployed. Used to construct a specific spack installation path, in conjunction with `spack-installs-root-path`. | `true` | N/A | `"0.21"` | -| `deployment-environment` | `string` | Name of the GitHub deployment target environment | `true` | N/A | `"Gadi Prerelease"` | +| `deployment-target` | `string` | Name of the GitHub deployment target | `true` | N/A | `"Gadi"` | +| `deployment-type` | `string` | Type of the GitHub deployment target | `true` | N/A | `"Release"`, `"Prerelease"` | | `spack-environment` | `string` | Spack environment name that is used for this deployment. Used to construct Prerelease spack-packages path, which is demarcated by the environment name | `true` | N/A | `"access-om2-pr12-12"` | ## Outputs @@ -29,7 +30,7 @@ This action constructs paths relevant to a deployment of `spack`. jobs: get-paths: runs-on: ubuntu-latest - environment: ${{ inputs.deployment-environment }} + environment: ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} steps: - name: Get Deployment Paths id: paths @@ -37,7 +38,8 @@ jobs: with: spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_PATH }} spack-version: "0.21" - deployment-environment: ${{ inputs.deployment-environment }} + deployment-target: ${{ inputs.deployment-target }} + deployment-type: ${{ inputs.deployment-type }} spack-environment: ${{ inputs.env-name }} - run: echo 'Spack is installed in `${{ steps.paths.outputs.spack }}` and spack-packages is installed in `${{ steps.paths.outputs.spack-packages }}`' diff --git a/.github/actions/get-deploy-paths/action.yml b/.github/actions/get-deploy-paths/action.yml index fdcb2ec5..ef06d5ee 100644 --- a/.github/actions/get-deploy-paths/action.yml +++ b/.github/actions/get-deploy-paths/action.yml @@ -9,21 +9,27 @@ inputs: spack-version: required: true description: | - Version of spack deployed. + Major branch version of spack deployed, eg. 0.22, 1.0. Used to construct a specific spack installation path, in conjunction with `spack-installs-root-path`. - deployment-environment: + deployment-target: required: true - description: Name of the GitHub deployment target environment + description: | + Name of the GitHub deployment target. Combined with inputs.deployment-type to form the GitHub Environment name. + For example: Gadi, Setonix + deployment-type: + required: true + description: | + Type of the GitHub deployment target. Combined with inputs.deployment-target to form the GitHub Environment name. + For example: Prerelease, Release spack-environment: required: true description: | Spack environment name that is used for this deployment. - Used to construct Prerelease spack-packages path, which is demarcated by the environment name. outputs: root: description: | Path to the root of a specific deployment of `spack`. - This path contains `spack-{packages,config}` repositories as well. + This path contains `spack-config` and various `spack-packages` repositories in it's hierarchy as well. value: ${{ steps.path.outputs.root }} spack: description: Path to a specific installation of `spack`. @@ -31,18 +37,12 @@ outputs: spack-config: description: Path to the ACCESS-NRI/spack-config repository associated with the install of spack. value: ${{ steps.path.outputs.spack-config }} - spack-packages-root: - description: | - Path to the folder containing all ACCESS-NRI/spack-packages repositories used in the install of spack. - For Release inputs.deployment-environment, this is the repository itself. - For Prerelease inputs.deployment-environment, this is the folder containing multiple spack-packages repositories. - value: ${{ steps.path.outputs.spack-packages-root }} - spack-packages: - description: Path to the ACCESS-NRI/spack-packages repository associated with the environment created in the install of spack. - value: ${{ steps.path.outputs.spack-packages }} spack-environment: - description: Path to the spack environment folder for the given inputs.spack-environment. + description: Path to the spack environment folder for the given inputs.spack-environment. Also contains the `spack-packages` repository for Prerelease deployments. value: ${{ steps.path.outputs.spack-environment }} + # Regarding our other important repository path, spack-packages: + # Release instances of spack-packages are now managed by spack itself via `spack repo` commands, so deployment doesn't actually need to know where it is. + # Prerelease instances of spack-packages are located within the outputs.spack-environment folder, and are removed when that environment is removed, so there is no need to track them either. runs: using: composite steps: @@ -54,19 +54,17 @@ runs: run: | # Check that the inputs.spack-installs-root-path exists - it's usually a var if [ -z '${{ inputs.spack-installs-root-path }}' ]; then - echo '::error::inputs.spack-installs-root-path does not exist in ${{ github.repository }}s ${{ inputs.deployment-environment }} environment. Check Environment vars.' + echo '::error::inputs.spack-installs-root-path does not exist in ${{ github.repository }}s ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} environment. Check Environment vars.' exit 1 fi - # We want the type of deployment environment, which is the last word in the string. - type=$(echo '${{ inputs.deployment-environment }}' | rev | cut -d ' ' -f 1 | rev) - if [[ ! '${{ env.ACCEPTABLE_DEPLOYMENT_ENVIRONMENT_TARGETS}}' =~ "$type" ]]; then - echo '::error::inputs.deployment-environment must be one of ${{ env.ACCEPTABLE_ENVIRONMENT_TARGETS }}.' + if [[ ! '${{ env.ACCEPTABLE_DEPLOYMENT_ENVIRONMENT_TARGETS}}' =~ "${{ inputs.deployment-type }}" ]]; then + echo '::error::inputs.deployment-type must be one of ${{ env.ACCEPTABLE_DEPLOYMENT_ENVIRONMENT_TARGETS }}.' exit 1 fi echo "type=$type" >> $GITHUB_OUTPUT - - name: Get ${{ inputs.deployment-environment }} Remote Paths + - name: Get ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} Remote Paths id: path shell: bash run: | @@ -74,12 +72,4 @@ runs: echo "root=$root" >> $GITHUB_OUTPUT echo "spack=$root/spack" >> $GITHUB_OUTPUT echo "spack-config=$root/spack-config" >> $GITHUB_OUTPUT - echo "spack-packages-root=$root/spack-packages" >> $GITHUB_OUTPUT - - if [[ '${{ steps.validate.outputs.type }}' == Release ]]; then - echo "spack-packages=$root/spack-packages" >> $GITHUB_OUTPUT - elif [[ '${{ steps.validate.outputs.type }}' == Prerelease ]]; then - echo "spack-packages=$root/spack-packages/${{ inputs.spack-environment }}/spack-packages" >> $GITHUB_OUTPUT - fi - echo "spack-environment=$root/environments/${{ inputs.spack-environment }}" >> $GITHUB_OUTPUT diff --git a/.github/actions/get-spack-manifest/README.md b/.github/actions/get-spack-manifest/README.md new file mode 100644 index 00000000..30dfff81 --- /dev/null +++ b/.github/actions/get-spack-manifest/README.md @@ -0,0 +1,38 @@ +# Get Spack Manifest Information + +Action that returns information about a Spack manifest file. + +## Inputs + +> [!NOTE] +> Action assumes that an appropriate repository is checked out prior to invocation + +| Name | Type | Description | Required | Default | Example | +| ---- | ---- | ----------- | -------- | ------- | ------- | +| `spack-manifest-path` | `string` | The path to the spack manifest file | `false` | `"spack.yaml"` | `"./some/other.spack.yaml"` | + +## Outputs + +| Name | Type | Description | Example | +| ---- | ---- | ----------- | ------- | +| `deployment-name` | `string` | The name of the deployment as specified in the reserved definition `_name` | `access-om2` | +| `deployment-version` | `string` | The version of the deployment as specified in the reserved definition `_version` | `2025.11.000` | + +## Example + +```yaml +# ... +jobs: + manifest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - id: spec + uses: access-nri/build-cd/.github/actions/get-spack-manifest@vX # for some version `vX` + with: + spack-manifest-path: spack.yaml + + - run: | + echo "Deploying ${{ steps.spec.outputs.deployment-name }} at ${{ steps.spec.outputs.deployment-version }}" +``` diff --git a/.github/actions/get-spack-manifest/action.yml b/.github/actions/get-spack-manifest/action.yml new file mode 100644 index 00000000..8076d0d2 --- /dev/null +++ b/.github/actions/get-spack-manifest/action.yml @@ -0,0 +1,52 @@ +name: Get Spack Manifest Information +description: Action that returns information about a Spack manifest file +author: Tommy Gatti +inputs: + spack-manifest-path: + required: false + default: spack.yaml + description: The path to the spack manifest file +outputs: + deployment-name: + description: | + The name of the deployment as specified in the reserved definition _name, in the spack manifest file. + value: ${{ steps.defs.outputs.name }} + deployment-version: + description: | + The version of the deployment as specified in the reserved definition _version, in the spack manifest file. + value: ${{ steps.defs.outputs.version }} +runs: + using: composite + steps: + - name: Clone build-cd script location + uses: actions/checkout@v4 + with: + repository: access-nri/build-cd + path: ${{ github.workspace }}/getter-script + + - name: Get Current Directory + id: script + # We need to store the current working directory so we can re-construct the inputs.spack-manifest-path, as we run + # the script from the build-cd directory + shell: bash + run: | + pwd=$(pwd) + echo "Action PWD: $pwd" + echo "pwd=$pwd" >> $GITHUB_OUTPUT + + - name: Get reserved definitions + id: defs + env: + PYTHONPATH: ${{ github.workspace }}/getter-script + shell: python + run: | + from scripts.spack_manifest.getter import ReservedDefinitions + import os + + defs = ReservedDefinitions.from_file("${{ steps.script.outputs.pwd }}/${{ inputs.spack-manifest-path }}") + name = defs.get("name") + version = defs.get("version") + + with open(os.environ['GITHUB_OUTPUT'], 'a') as o: + o.write(f"name={name}\n") + o.write(f"version={version}\n") diff --git a/.github/actions/get-spack-root-spec/README.md b/.github/actions/get-spack-root-spec/README.md deleted file mode 100644 index c4c29f38..00000000 --- a/.github/actions/get-spack-root-spec/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Get Spack Manifest Information - -Action that returns information about a Spack manifest file. - -## Inputs - -> [!NOTE] -> Action assumes that an appropriate repository is checked out prior to invocation - -| Name | Type | Description | Required | Default | Example | -| ---- | ---- | ----------- | -------- | ------- | ------- | -| `spack-manifest-path` | `string` | The path to the spack manifest file | `false` | `"./spack.yaml"` | `"./some/other.spack.yaml"` | - -## Outputs - -| Name | Type | Description | Example | -| ---- | ---- | ----------- | ------- | -| `root-spec` | `string` | The entirety of the root spec in the spack manifest file | `"access-om2@git.2025.01.01=release ~deterministic"` | -| `root-spec-name` | `string` | The name of the root spec in the spack manifest file | `"access-om2"` | -| `root-spec-ref` | `string` | The git ref from the root spec in the spack manifest file | `"2025.01.01"` | -| `root-spec-version` | `string` | The spack version from the root spec in the spack manifest file | `"release"` | -| `yq-root-spec` | `string` (`yq` filter) | The yq filter for obtaining the root spec for the spack manifest file - may change based on whether the root spec is in multi-target or single-target format | `.spack.specs[0]` | - -## Example - -```yaml -# ... -jobs: - manifest: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - id: spec - uses: access-nri/build-cd/.github/actions/get-spack-root-spec@vX # for some version `vX` - with: - spack-manifest-path: ./spack.yaml - - - run: | - echo "Deploying ${{ steps.spec.outputs.root-spec-name }} at ${{ steps.spec.outputs.root-spec-version }}" - - # You can tack on more filters on top of the `root-spec` one - variants=$(yq '${{ steps.spec.outputs.yq-root-spec }} | capture("[^~+- ]+(.+)") | .[0]' ./spack.yaml) - echo "Variants are $variants" -``` diff --git a/.github/actions/get-spack-root-spec/action.yml b/.github/actions/get-spack-root-spec/action.yml deleted file mode 100644 index 6f8c438c..00000000 --- a/.github/actions/get-spack-root-spec/action.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: Get Spack Manifest Information -description: Action that returns information about a Spack manifest file -author: Tommy Gatti -inputs: - spack-manifest-path: - required: false - default: ./spack.yaml - description: The path to the spack manifest file -outputs: - root-spec: - description: | - The entirety of the root spec in the spack manifest file. - For example: 'access-om2@git.2025.01.01=access-om2 ~deterministic'. - value: ${{ steps.spec.outputs.full }} - root-spec-name: - description: | - The name of the root spec in the spack manifest file. - For example: 'access-om2'. - value: ${{ steps.spec.outputs.name }} - root-spec-ref: - description: | - The git ref from the root spec in the spack manifest file. - For example: '2025.01.01'. - value: ${{ steps.spec.outputs.ref }} - root-spec-version: - description: | - The spack version from the root spec in the spack manifest file. - For example, 'release'. - value: ${{ steps.spec.outputs.version }} - root-spec-variants: - description: | - The variants from the root spec in the spack manifest file. - For example: '~deterministic'. - value: ${{ steps.spec.outputs.variants }} - yq-root-spec: - description: | - The yq filter for the root spec of the spack manifest file. - value: ${{ steps.yq.outputs.filter }} -runs: - using: composite - steps: - - name: Set yq filter for root spec - id: yq - shell: bash - # This attempts to get the spack.yaml model first via the multi-target - # .spack.definitions.root_package[0] method, then the traditional '.spack.specs[0]'. - env: - MULTI_TARGET_ROOT_SPEC_YQ_FILTER: (.spack.definitions[] | select(."ROOT_PACKAGE") | .[][0]) - SINGLE_TARGET_ROOT_SPEC_YQ_FILTER: .spack.specs[0] - run: | - if [[ -n "$(yq '${{ env.MULTI_TARGET_ROOT_SPEC_YQ_FILTER }}' ${{ inputs.spack-manifest-path }})" ]]; then - filter="${{ env.MULTI_TARGET_ROOT_SPEC_YQ_FILTER }}" - elif [[ -n "$(yq '${{ env.SINGLE_TARGET_ROOT_SPEC_YQ_FILTER }}' ${{ inputs.spack-manifest-path }})" ]]; then - filter="${{ env.SINGLE_TARGET_ROOT_SPEC_YQ_FILTER }}" - else - echo "We can't find the root spec!" - exit 1 - fi - - echo "Based on the root spec, we are using '$filter'" - echo "filter=$filter" >> $GITHUB_OUTPUT - - - name: Get spec info from spack manifest - id: spec - shell: bash - run: | - # Example: access-om2@git.2025.01.0=release ~variant +debug some=value - full=$(yq '${{ steps.yq.outputs.filter }}' ${{ inputs.spack-manifest-path }}) - - # Example of captured groups from the above: - # name: anything before `@`. Ex: access-om2 - # ref: anything after `@` or `@git.`; but before `=` (for =VERSION syntax) or ` `/`+`/`~` (for variants). Ex: 2025.01.0 - # version: anything after a `=`, but before ` `/`+`/`~` (for variants). Ex: release - # variants: anything at the end that contains ` `/`+`/`~` Ex. ~variant +debug some=value - - groups_regex='(?.+)@(?:git\\.)?(?[^=+~ ]+)(?:=(?[^~+ ]+))?(?[~+ ].+)?' - - groups=$(yq '${{ steps.yq.outputs.filter }} | capture("'"$groups_regex"'")' ${{ inputs.spack-manifest-path }}) - - # Pull values from groups above - name=$(yq '.name' <<< "$groups") - ref=$(yq '.ref' <<< "$groups") - version=$(yq '.version' <<< "$groups") - variants=$(yq '.variants' <<< "$groups") - - echo "Split '$full' into name: '$name', ref: '$ref', version: '$version', variants: '$variants'" - - echo "full=$full" >> $GITHUB_OUTPUT - echo "name=$name" >> $GITHUB_OUTPUT - echo "ref=$ref" >> $GITHUB_OUTPUT - echo "version=$version" >> $GITHUB_OUTPUT - echo "variants=$variants" >> $GITHUB_OUTPUT diff --git a/.github/actions/validate-repo-version/README.md b/.github/actions/validate-repo-version/README.md deleted file mode 100644 index 1d9d4ff0..00000000 --- a/.github/actions/validate-repo-version/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Validate Repo Versions Action - -This action checks that the tags specified in a models `config/versions.json` is an actual, valid tag for that repository. - -## Inputs - -| Name | Type | Description | Required | Default | Example | -| ---- | ---- | ----------- | -------- | ------- | ------- | -| `repo-to-check` | `string` | ACCESS-NRI repository to validate associated version in `config/versions.json` | `true` | N/A | `spack-packages` | -| `pr` | `number` | The pull request number that contains the `config/versions.json` | `true` | N/A | 12 | - -## Outputs - -| Name | Type | Description | Example | -| ---- | ---- | ----------- | ------- | -| `version` | `string` | Version of the repo from the `config/versions.json` | `"2024.03.12"` | -| `git-hash` | `string` (hash) | Git hash of the version of the repo from the `config/versions.json` | `"rn34rj2e8ue2928rhwjfehiwfu2yrfhwieuhriwh"` | - -## Example - -```yaml -# ... -- id: validate - uses: access-nri/build-cd/.github/actions/validate-repo-version@vX - with: - repo-to-check: spack-packages - pr: 12 - -- run: echo "spack-packages has valid version ${{ steps.validate.outputs.version }} (${{ steps.validate.outputs.git-hash }}) in PR#12's `config/versions.json`" - -- if: failure() && steps.validate.outcome == 'failure' - run: echo "The version in spack-packages is not valid." -``` diff --git a/.github/actions/validate-repo-version/action.yml b/.github/actions/validate-repo-version/action.yml deleted file mode 100644 index 31f573d2..00000000 --- a/.github/actions/validate-repo-version/action.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Validate Repo Version -description: Action that validates that a version in `config/versions.json` matches the version in the repository -inputs: - repo-to-check: - required: true - description: ACCESS-NRI repository to validate associated version in `config/versions.json` - pr: - required: true - description: The pull request number that contains the config/versions.json -outputs: - version: - value: ${{ steps.jq.outputs.version }} - description: Version of `inputs.repo-to-check` from the `config/versions.json` - git-hash: - value: ${{ steps.get-git-hash.outputs.git-hash }} - description: The git hash of the `inputs.repo-to-check` at the specified version -runs: - using: composite - steps: - # Checkout the callers `config/versions.json` - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr }} - - # Get the version from the `config/versions.json` - - name: Setup - id: jq - shell: bash - run: | - version=$(jq --compact-output --raw-output '."${{ inputs.repo-to-check }}"' ./config/versions.json) - - if [[ "${version}" == "null" ]]; then - echo "::error::There is no `${{ inputs.repo-to-check }}` in `./config/versions.json`" - exit 1 - fi - - echo "version=${version}" >> $GITHUB_OUTPUT - echo "spack-branch-version=releases/v${version}" >> $GITHUB_OUTPUT - - # Verify that the repository exists at the given ref - - name: Version Check - id: check - uses: actions/checkout@v4 - with: - repository: access-nri/${{ inputs.repo-to-check }} - # In the case where we are checking spack, we need to add the - # 'releases/v' to the ref name, obtained in the previous step - ref: ${{ inputs.repo-to-check == 'spack' && steps.jq.outputs.spack-branch-version || steps.jq.outputs.version }} - path: repo - - - name: Get Git Hash - id: get-git-hash - shell: bash - run: echo "git-hash=$(git -C repo rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Version Check Failure Notifier - if: failure() && steps.check.outcome == 'failure' - shell: bash - run: | - echo "::error::`${{ inputs.repo-to-check }}` at the specified ref (`${{ steps.jq.outputs.version }}`) doesn't exist." - exit 1 diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 28dcc7b5..09a8ca96 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -9,21 +9,14 @@ on: model: type: string required: true - description: The model that is being tested and deployed - root-sbd: - type: string - required: false - # default: The ${{ inputs.model }} above - description: | - The name of the root Spack Bundle Definition, if it is different from the model name. - This is often a package named similarly in ACCESS-NRI/spack-packages. + description: The human-readable model name that is being tested and deployed spack-manifest-path: type: string required: false - default: ./spack.yaml + default: spack.yaml description: | The Spack manifest path relative to the caller repository that will be used for the deployment. - This is usually `./spack.yaml`, but can be overridden if needed. + This is usually `spack.yaml`, but can be overridden if needed. tag-deployment: type: boolean required: false @@ -72,26 +65,24 @@ env: METADATA_PATH: /opt/metadata OUTPUTS_PATH: /opt/outputs jobs: - defaults: - name: Set Defaults - # Unfortunately, you can't set a dynamic default value based on `inputs` yet + init: + name: Initialize Deployment runs-on: ubuntu-latest outputs: - root-sbd: ${{ steps.root-sbd.outputs.default }} + deployment-name: ${{ steps.manifest.outputs.deployment-name }} targets: ${{ steps.target.outputs.valid-targets }} steps: - - name: root-sbd - id: root-sbd - run: | - if [ -z "${{ inputs.root-sbd }}" ]; then - echo "default=${{ inputs.model }}" >> $GITHUB_OUTPUT - else - echo "default=${{ inputs.root-sbd }}" >> $GITHUB_OUTPUT - fi + - uses: actions/checkout@v4 + + - name: Get manifest + id: manifest + uses: access-nri/build-cd/.github/actions/get-spack-manifest@v8 + with: + spack-manifest-path: ${{ inputs.spack-manifest-path }} - name: Generate Deployment Target Matrix id: target - uses: access-nri/build-cd/.github/actions/get-target-matrix@v7 + uses: access-nri/build-cd/.github/actions/get-target-matrix@v8 with: targets: ${{ vars.RELEASE_DEPLOYMENT_TARGETS }} @@ -99,34 +90,34 @@ jobs: name: Verify Deployment Settings runs-on: ubuntu-latest needs: - - defaults + - init strategy: matrix: - target: ${{ fromJson(needs.defaults.outputs.targets) }} + target: ${{ fromJson(needs.init.outputs.targets) }} steps: - uses: actions/checkout@v4 with: repository: access-nri/build-cd - - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v7 + - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v8 with: settings-path: ./config/settings.json target: ${{ matrix.target }} error-level: error - get-root-spec-ref: - name: Get root spec ref + get-deployment-version: + name: Get Deployment Version runs-on: ubuntu-latest needs: - verify-settings outputs: - root-spec-ref: ${{ steps.tag.outputs.root-spec-ref }} + version: ${{ steps.manifest.outputs.deployment-version }} steps: - uses: actions/checkout@v4 - - name: Get root spec ref - id: tag - uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7 + - name: Get Deployment Version + id: manifest + uses: access-nri/build-cd/.github/actions/get-spack-manifest@v8 with: spack-manifest-path: ${{ inputs.spack-manifest-path }} @@ -135,7 +126,7 @@ jobs: if: inputs.tag-deployment runs-on: ubuntu-latest needs: - - get-root-spec-ref + - get-deployment-version permissions: contents: write steps: @@ -155,39 +146,34 @@ jobs: - name: Push Tag env: - TAG: ${{ needs.get-root-spec-ref.outputs.root-spec-ref }} + TAG: ${{ needs.get-deployment-version.outputs.version }} run: | - if [[ "${{ env.TAG }}" == "latest" ]]; then - echo "::error::The version 'latest' is reserved for a spack package that moves often and cannot be used as a release tag. Reset the 'main' or 'backport' branch, reopen the merged PR, and update the version." - exit 1 - fi - git tag ${{ env.TAG }} -m "Deployment of ${{ inputs.model }} ${{ env.TAG }} via build-cd 'cd.yml' workflow" git push --tags deploy-release: name: Deploy Release needs: - - defaults - - get-root-spec-ref + - init + - get-deployment-version - push-tag # Pushing tags is optional if we are doing a non-model deployment if: >- always() && - needs.defaults.result == 'success' && - needs.get-root-spec-ref.result == 'success' && + needs.init.result == 'success' && + needs.get-deployment-version.result == 'success' && (needs.push-tag.result == 'success' || needs.push-tag.result == 'skipped') strategy: matrix: - target: ${{ fromJson(needs.defaults.outputs.targets) }} - uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v7 + target: ${{ fromJson(needs.init.outputs.targets) }} + uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v8 with: deployment-target: ${{ matrix.target }} deployment-ref: ${{ github.ref_name }} deployment-type: Release - deployment-version: ${{ needs.get-root-spec-ref.outputs.root-spec-ref }} + deployment-version: ${{ needs.get-deployment-version.outputs.version }} spack-manifest-path: ${{ inputs.spack-manifest-path }} - expected-root-spec-name: ${{ needs.defaults.outputs.root-sbd }} + expected-root-spec-name: ${{ needs.init.outputs.deployment-name }} spack-manifest-schema-path: ${{ inputs.spack-manifest-schema-path }} spack-manifest-schema-version: ${{ inputs.spack-manifest-schema-version }} config-versions-schema-version: ${{ inputs.config-versions-schema-version }} @@ -203,8 +189,8 @@ jobs: name: Create Release if: inputs.tag-deployment needs: - - defaults - - get-root-spec-ref + - init + - get-deployment-version - push-tag - deploy-release runs-on: ubuntu-latest @@ -245,8 +231,8 @@ jobs: id: release-body env: J2_MODEL: ${{ inputs.model }} - J2_VERSION: ${{ needs.get-root-spec-ref.outputs.root-spec-ref }} - J2_ROOT_SBD: ${{ needs.defaults.outputs.root-sbd }} + J2_VERSION: ${{ needs.get-deployment-version.outputs.version }} + J2_ROOT_SBD: ${{ needs.init.outputs.deployment-name }} run: | python -m scripts.jinja_template.render_deployment_info \ --template scripts/jinja_template/templates/release-body.md.j2 \ @@ -256,8 +242,8 @@ jobs: - name: Create Release uses: softprops/action-gh-release@69320dbe05506a9a39fc8ae11030b214ec2d1f87 # v2.0.5 with: - tag_name: ${{ needs.get-root-spec-ref.outputs.root-spec-ref }} - name: ${{ inputs.model}} ${{ needs.get-root-spec-ref.outputs.root-spec-ref }} + tag_name: ${{ needs.get-deployment-version.outputs.version }} + name: ${{ inputs.model}} ${{ needs.get-deployment-version.outputs.version }} body_path: ${{ env.TEMPLATED_RELEASE_BODY_PATH }} generate_release_notes: true fail_on_unmatched_files: true @@ -270,7 +256,7 @@ jobs: name: Build DB Metadata Upload if: inputs.tag-deployment && inputs.upload-to-build-db needs: - - defaults + - init - deploy-release - release runs-on: ubuntu-latest @@ -313,7 +299,7 @@ jobs: run: | python -m scripts.release_provenance.tracking_services_data \ --repository ${{ github.repository }} \ - --root-spec ${{ needs.defaults.outputs.root-sbd }} \ + --root-spec ${{ needs.init.outputs.deployment-name }} \ --deployment-outputs ${{ env.OUTPUTS_PATH }} \ --metadata-outputs ${{ env.METADATA_PATH }} \ --upload diff --git a/.github/workflows/ci-closed.yml b/.github/workflows/ci-closed.yml index d96aedb3..6b0d9c9b 100644 --- a/.github/workflows/ci-closed.yml +++ b/.github/workflows/ci-closed.yml @@ -1,5 +1,5 @@ name: PR Closed Cleanup -run-name: ${{ inputs.model }} PR Closed Cleanup +run-name: PR Closed Cleanup # Remove prereleases that were part of a closed PR, so we save space # on our deployment targets. If needed, one can still get the # spack.yaml as part of the closed PR and revive it themselves. @@ -9,10 +9,10 @@ run-name: ${{ inputs.model }} PR Closed Cleanup on: workflow_call: inputs: - root-sbd: + spack-manifest-path: type: string - required: true - description: The model package name that is going to be removed + required: false + default: spack.yaml # Callers usually have the trigger: # pull_request: # types: @@ -31,14 +31,23 @@ jobs: version-pattern: ${{ steps.version.outputs.pattern }} targets: ${{ steps.target.outputs.valid-targets }} steps: + - name: Get data from ${{ github.repository }} + uses: actions/checkout@v4 + + - name: Get Deployment Name + id: manifest + uses: access-nri/build-cd/.github/actions/get-spack-manifest@v8 + with: + spack-manifest-path: ${{ inputs.spack-manifest-path }} + - name: Get Version Pattern id: version # For example, `access-om3-pr12-*` - run: echo "pattern=${{ inputs.root-sbd }}-pr${{ github.event.pull_request.number }}-*" >> $GITHUB_OUTPUT + run: echo "pattern=${{ steps.manifest.outputs.deployment-name }}-pr${{ github.event.pull_request.number }}-*" >> $GITHUB_OUTPUT - name: Generate Deployment Target Matrix id: target - uses: access-nri/build-cd/.github/actions/get-target-matrix@v7 + uses: access-nri/build-cd/.github/actions/get-target-matrix@v8 with: targets: ${{ vars.PRERELEASE_DEPLOYMENT_TARGETS }} @@ -50,9 +59,9 @@ jobs: matrix: target: ${{ fromJson(needs.setup.outputs.targets) }} fail-fast: false - uses: access-nri/build-cd/.github/workflows/undeploy-1-start.yml@v7 + uses: access-nri/build-cd/.github/workflows/undeploy-1-start.yml@v8 with: - version-pattern: ${{ inputs.root-sbd }}-pr${{ github.event.pull_request.number }}-* + version-pattern: ${{ needs.setup.outputs.version-pattern }} target: ${{ matrix.target }} type: Prerelease secrets: inherit diff --git a/.github/workflows/ci-command-configs.yml b/.github/workflows/ci-command-configs.yml index 3f1d97cc..3e7bb379 100644 --- a/.github/workflows/ci-command-configs.yml +++ b/.github/workflows/ci-command-configs.yml @@ -2,18 +2,15 @@ name: Configs on: workflow_call: inputs: + spack-manifest-path: + type: string + required: false + default: spack.yaml + description: Path to the spack manifest in the model deployment repository model: type: string required: true description: The model that is being tested and deployed - root-sbd: - type: string - required: false - # The equivalent default is set in the defaults job below. - # default: ${{ inputs.model }} - description: | - The name of the root Spack Bundle Definition, if it is different from the model name. - This is often a package named similarly in ACCESS-NRI/spack-packages. auto-configs-pr-schema-version: type: string required: true @@ -41,8 +38,8 @@ jobs: # and read configuration information from the caller repository. runs-on: ubuntu-latest outputs: - # Value for the defaulted root-sbd if not provided - root-sbd: ${{ steps.defaults.outputs.root-sbd }} + # Deployment Name for the spack manifest - corresponds to the root spec + deployment-name: ${{ steps.manifest.outputs.deployment-name }} # Caller PR branch ref pr-branch: ${{ steps.pr.outputs.ref }} # The profile to use from the MDRs config/auto-configs-pr.json @@ -104,15 +101,6 @@ jobs: # Output processed args echo "profile=$profile" >> $GITHUB_OUTPUT - - name: Set defaults - id: defaults - run: | - if [[ "${{ inputs.root-sbd }}" == "" ]]; then - echo "root-sbd=${{ inputs.model }}" >> $GITHUB_OUTPUT - else - echo "root-sbd=${{ inputs.root-sbd }}" >> $GITHUB_OUTPUT - fi - - name: Get caller repository ref id: pr # We need to find the PR ref of the caller repository explicitly, as this workflow trigger (issue_comment) is in the context of the default branch @@ -122,26 +110,25 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ steps.pr.outputs.ref }} - path: caller - name: Validate auto configs PR configuration uses: access-nri/schema/.github/actions/validate-with-schema@main with: schema-version: ${{ inputs.auto-configs-pr-schema-version }} schema-location: au.org.access-nri/model/deployment/config/auto-configs-pr - data-location: caller/config/auto-configs-pr.json + data-location: config/auto-configs-pr.json - name: Read auto configs PR configuration id: read-config run: | - if ! jq --exit-status '.profiles."${{ steps.parse.outputs.profile }}"' caller/config/auto-configs-pr.json; then + if ! jq --exit-status '.profiles."${{ steps.parse.outputs.profile }}"' config/auto-configs-pr.json; then echo "::error::Profile ${{ steps.parse.outputs.profile }} does not exist in config/auto-configs-pr.json at ref ${{ steps.pr.outputs.ref }}" exit 1 fi - configs=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs | keys' caller/config/auto-configs-pr.json) - configs_formatted=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs | keys | join(" ")' caller/config/auto-configs-pr.json) - configs_repo=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs_repo' caller/config/auto-configs-pr.json) + configs=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs | keys' config/auto-configs-pr.json) + configs_formatted=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs | keys | join(" ")' config/auto-configs-pr.json) + configs_repo=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs_repo' config/auto-configs-pr.json) echo "$configs" echo "$configs_formatted" @@ -151,6 +138,12 @@ jobs: echo "configs-formatted=$configs_formatted" >> $GITHUB_OUTPUT echo "configs-repo=$configs_repo" >> $GITHUB_OUTPUT + - name: Get caller deployment name + id: manifest + uses: access-nri/build-cd/.github/actions/get-spack-manifest@v8 + with: + spack-manifest-path: ${{ inputs.spack-manifest-path }} + update-configs: name: Update # Open PRs in the callers configs repository with the updated prerelease build @@ -223,7 +216,7 @@ jobs: - name: Open PR id: open-pr env: - DEPLOYMENT_IDENTIFIER: ${{ needs.setup.outputs.root-sbd }}/pr${{ github.event.issue.number }}-${{ steps.previous-deployments.outputs.deployments }} + DEPLOYMENT_IDENTIFIER: ${{ needs.setup.outputs.deployment-name }}/pr${{ github.event.issue.number }}-${{ steps.previous-deployments.outputs.deployments }} run: | pr_branch_name="auto/pr${{ github.event.issue.number }}-${{ steps.previous-deployments.outputs.deployments }}/${{ env.CONFIG }}" @@ -250,7 +243,7 @@ jobs: PYTHONPATH=build-cd python3 -m scripts.model_config_manifest.prerelease_update \ --manifest caller-configs/config.yaml \ --deployment-target Gadi \ - --root-sbd ${{ needs.setup.outputs.root-sbd }} \ + --root-sbd ${{ needs.setup.outputs.deployment-name }} \ --module ${{ env.DEPLOYMENT_IDENTIFIER }} echo "Committing and pushing changes..." diff --git a/.github/workflows/ci-comment.yml b/.github/workflows/ci-comment.yml index 64e1565f..226aff12 100644 --- a/.github/workflows/ci-comment.yml +++ b/.github/workflows/ci-comment.yml @@ -5,8 +5,6 @@ run-name: ${{ inputs.model }} Comment Command # permissions.contents:write # permissions.pull-requests:write -# FIXME: We don't support any !bump from a spack manifest that is not in the default ./spack.yaml location. - on: workflow_call: inputs: @@ -14,14 +12,11 @@ on: type: string required: true description: The model that is being tested and deployed - root-sbd: + spack-manifest-path: type: string required: false - # The equivalent default is set in the defaults job below. - # default: ${{ inputs.model }} - description: | - The name of the root Spack Bundle Definition, if it is different from the model name. - This is often a package named similarly in ACCESS-NRI/spack-packages. + default: spack.yaml + description: Path the the spack manifest in the caller # Callers usually have the trigger: # issue_comment: # types: @@ -30,34 +25,15 @@ on: env: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} jobs: - defaults: - name: Set Defaults - # Unfortunately, you can't set a dynamic default value based on `inputs` yet. - runs-on: ubuntu-latest - outputs: - root-sbd: ${{ steps.root-sbd.outputs.default }} - steps: - - name: root-sbd default - id: root-sbd - run: | - if [[ "${{ inputs.root-sbd }}" == "" ]]; then - echo "default=${{ inputs.model }}" >> $GITHUB_OUTPUT - else - echo "default=${{ inputs.root-sbd }}" >> $GITHUB_OUTPUT - fi - bump-version: name: Bump spack.yaml if: github.event.issue.pull_request && startsWith(github.event.comment.body, '!bump') - needs: - - defaults runs-on: ubuntu-latest permissions: pull-requests: write contents: write env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SPACK_YAML_MODEL_PROJECTION_YQ: .spack.modules.default.tcl.projections.${{ needs.defaults.outputs.root-sbd }} steps: - name: Get branch from issue id: issue @@ -73,7 +49,9 @@ jobs: - name: Original version id: original - uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7 + uses: access-nri/build-cd/.github/actions/get-spack-manifest@v8 + with: + spack-manifest-path: ${{ inputs.spack-manifest-path }} - name: Setup id: setup @@ -99,11 +77,11 @@ jobs: echo "version=${tag_date}" >> $GITHUB_OUTPUT echo "bump=major" >> $GITHUB_OUTPUT else - echo "version=${{ steps.original.outputs.root-spec-ref }}" >> $GITHUB_OUTPUT + echo "version=${{ steps.original.outputs.deployment-version }}" >> $GITHUB_OUTPUT echo "bump=current" >> $GITHUB_OUTPUT fi elif [[ "${{ contains(github.event.comment.body, 'minor')}}" == "true" ]]; then - echo "version=${{ steps.original.outputs.root-spec-ref }}" >> $GITHUB_OUTPUT + echo "version=${{ steps.original.outputs.deployment-version }}" >> $GITHUB_OUTPUT echo "bump=minor" >> $GITHUB_OUTPUT else echo "::warning::Usage: `!bump [major|minor]`, got `${{ github.event.comment.body }}`" @@ -131,26 +109,21 @@ jobs: git_tag_gpgsign: true - name: Update, Commit and Push the Bump - env: - # If the root spec contained an '=VERSION' segment, we want to add that back to the updated root spec, too - OPTIONAL_VERSION: ${{ steps.original.outputs.root-spec-version != '' && format('={0}', steps.original.outputs.root-spec-version) || '' }} run: | - updated_spec="${{ needs.defaults.outputs.root-sbd }}@git.${{ steps.bump.outputs.after }}${{ env.OPTIONAL_VERSION }}" - yq -i "${{ steps.original.outputs.yq-root-spec }} = \"$updated_spec\"" spack.yaml - yq -i '${{ env.SPACK_YAML_MODEL_PROJECTION_YQ }} = "{name}/${{ steps.bump.outputs.after }}"' spack.yaml - git add spack.yaml - git commit -m "spack.yaml: Updated ${{ needs.defaults.outputs.root-sbd }} package version from ${{ steps.original.outputs.root-spec-ref }} to ${{ steps.bump.outputs.after }}" + yq -i '.spack.definitions[]._version[0] = "${{ steps.bump.outputs.after }}"' ${{ inputs.spack-manifest-path }} + git add ${{ inputs.spack-manifest-path }} + git commit -m "${{ inputs.spack-manifest-path }}: Updated deployment version from ${{ steps.original.outputs.deployment-version }} to ${{ steps.bump.outputs.after }}" git push - name: Success Notifier uses: access-nri/actions/.github/actions/pr-comment@main with: comment: | - :white_check_mark: Version bumped from `${{ steps.original.outputs.root-spec-ref }}` to `${{ steps.bump.outputs.after }}` :white_check_mark: + :white_check_mark: ${{ inputs.spack-manifest-path }} version bumped from `${{ steps.original.outputs.deployment-version }}` to `${{ steps.bump.outputs.after }}` :white_check_mark: - name: Failure Notifier if: failure() uses: access-nri/actions/.github/actions/pr-comment@main with: comment: | - :x: Failed to bump version or commit changes, see ${{ env.RUN_URL }} :x: + :x: Failed to bump version or commit changes to ${{ inputs.spack-manifest-path }}, see ${{ env.RUN_URL }} :x: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aff66093..b5067e3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,24 +15,13 @@ on: type: string required: true description: The model that is being tested and deployed - root-sbd: - type: string - required: false - # default: The ${{ inputs.model }} above - description: | - The name of the root Spack Bundle Definition, if it is different from the model name. - This is often a package named similarly in ACCESS-NRI/spack-packages. - pr: - type: string - required: true - description: The pull request number that will be deployed as a prerelease spack-manifest-path: type: string required: false - default: ./spack.yaml + default: spack.yaml description: | The Spack manifest path relative to the caller repository that will be used for the deployment. - This is usually `./spack.yaml`, but can be overridden if needed. + This is usually `spack.yaml`, but can be overridden if needed. spack-manifest-schema-path: type: string required: false @@ -68,17 +57,17 @@ on: # - edited env: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} jobs: - defaults: - name: Set Defaults - # Unfortunately, you can't set a dynamic default value based on `inputs` yet. - # We also set the PR and branch metadata here because it's used in multiple places, + init: + name: Setup CI + # We set the PR and branch metadata here because it's used in multiple places, # including the deploy reusable workflow, which can't access the `env` context. runs-on: ubuntu-latest env: GH_TOKEN: ${{ github.token }} outputs: - root-sbd: ${{ steps.root-sbd.outputs.default }} + deployment-name: ${{ steps.manifest.outputs.deployment-name }} head-ref: ${{ steps.pr.outputs.head }} head-sha: ${{ steps.pr.outputs.sha }} base-ref: ${{ steps.pr.outputs.base }} @@ -86,21 +75,12 @@ jobs: prerelease-version: ${{ steps.prerelease.outputs.version }} targets: ${{ steps.target.outputs.valid-targets }} steps: - - name: root-sbd default - id: root-sbd - run: | - if [[ "${{ inputs.root-sbd }}" == "" ]]; then - echo "default=${{ inputs.model }}" >> $GITHUB_OUTPUT - else - echo "default=${{ inputs.root-sbd }}" >> $GITHUB_OUTPUT - fi - - name: PR metadata id: pr run: | - pr_metadata=$(gh pr view ${{ inputs.pr }} --repo ${{ github.repository }} --json headRefName,headRefOid,baseRefName) + pr_metadata=$(gh pr view ${{ env.PR_NUMBER }} --repo ${{ github.repository }} --json headRefName,headRefOid,baseRefName) if [ -z "$pr_metadata" ]; then - echo "::error::Failed to get PR ${{ inputs.pr }} metadata." + echo "::error::Failed to get PR ${{ env.PR_NUMBER }} metadata." exit 1 fi @@ -117,7 +97,7 @@ jobs: '$pr.baseRefName' ) - echo "PR ${{ inputs.pr }} with '$head' ('$sha') -> '$base'" + echo "PR ${{ env.PR_NUMBER }} with '$head' ('$sha') -> '$base'" echo "head=$head" >> $GITHUB_OUTPUT echo "sha=$sha" >> $GITHUB_OUTPUT echo "base=$base" >> $GITHUB_OUTPUT @@ -126,7 +106,7 @@ jobs: id: previous-deploys uses: access-nri/build-cd/.github/actions/get-pr-deployments@v7 with: - pr: ${{ inputs.pr }} + pr: ${{ env.PR_NUMBER }} - name: Branch metadata id: prerelease @@ -138,21 +118,31 @@ jobs: echo "Next Deployment Number is ${{ steps.previous-deploys.outputs.deployments }} + $next_deployment_is_pr_deployment = $next_deployment_number" echo "next-deployment-number=$next_deployment_number" >> $GITHUB_OUTPUT - version="pr${{ inputs.pr }}-$next_deployment_number" + version="pr${{ env.PR_NUMBER }}-$next_deployment_number" echo "Prerelease version will be $version" echo "version=$version" >> $GITHUB_OUTPUT - name: Generate Deployment Target Matrix id: target - uses: access-nri/build-cd/.github/actions/get-target-matrix@v7 + uses: access-nri/build-cd/.github/actions/get-target-matrix@v8 with: targets: ${{ vars.PRERELEASE_DEPLOYMENT_TARGETS }} + - uses: actions/checkout@v4 + with: + ref: ${{ steps.pr.outputs.head }} + + - name: Get Deployment Name from Spack Manifest + id: manifest + uses: access-nri/build-cd/.github/actions/get-spack-manifest@v8 + with: + spack-manifest-path: ${{ inputs.spack-manifest-path }} + redeploy-pre: name: '!redeploy Pending' if: github.event_name == 'issue_comment' && startsWith(github.event.comment.body, '!redeploy') needs: - - defaults + - init runs-on: ubuntu-latest permissions: statuses: write @@ -188,38 +178,38 @@ jobs: # We don't want to use history expansion (the '!') shell: bash +H {0} run: | - echo 'context=!redeploy Number ${{ needs.defaults.outputs.next-deployment-number }}' >> $GITHUB_OUTPUT + echo 'context=!redeploy Number ${{ needs.init.outputs.next-deployment-number }}' >> $GITHUB_OUTPUT echo 'description=Redeploy Prerelease' >> $GITHUB_OUTPUT - name: Set Commit Status Pending uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: status: pending - sha: ${{ needs.defaults.outputs.head-sha }} + sha: ${{ needs.init.outputs.head-sha }} context: ${{ steps.commit-status-args.outputs.context }} description: ${{ steps.commit-status-args.outputs.description }} deploy: name: Deploy needs: - - defaults + - init strategy: fail-fast: false matrix: # Example: ['Gadi', 'Setonix', ...] - target: ${{ fromJson(needs.defaults.outputs.targets) }} - uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v7 + target: ${{ fromJson(needs.init.outputs.targets) }} + uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v8 with: deployment-target: ${{ matrix.target }} - deployment-ref: ${{ needs.defaults.outputs.head-ref }} + deployment-ref: ${{ needs.init.outputs.head-ref }} deployment-type: Prerelease # The prerelease looks like: `pr-`. # Ex. Pull Request #12 with 2 deployments on branch -> `pr12-2`. # Note that for multi-target prereleases, they share the same deployment number. - deployment-version: ${{ needs.defaults.outputs.prerelease-version }} - prerelease-compare-ref: ${{ needs.defaults.outputs.base-ref }} + deployment-version: ${{ needs.init.outputs.prerelease-version }} + prerelease-compare-ref: ${{ needs.init.outputs.base-ref }} spack-manifest-path: ${{ inputs.spack-manifest-path }} - expected-root-spec-name: ${{ needs.defaults.outputs.root-sbd }} + expected-root-spec-name: ${{ needs.init.outputs.deployment-name }} # Pass versions and paths for schemas into the deploy workflow spack-manifest-schema-path: ${{ inputs.spack-manifest-schema-path }} spack-manifest-schema-version: ${{ inputs.spack-manifest-schema-version }} @@ -234,7 +224,7 @@ jobs: # failed, skipped, cancelled redeploy = failure commit status if: always() && needs.redeploy-pre.result == 'success' needs: - - defaults # to get access to the head-sha + - init # to get access to the head-sha - redeploy-pre # to get the initial commit status context and description - deploy # to get the overall status of this workflow runs-on: ubuntu-latest @@ -252,7 +242,7 @@ jobs: uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: status: ${{ needs.deploy.result == 'success' && 'success' || 'failure' }} - sha: ${{ needs.defaults.outputs.head-sha }} + sha: ${{ needs.init.outputs.head-sha }} context: ${{ needs.redeploy-pre.outputs.commit-status-context }} description: ${{ needs.redeploy-pre.outputs.commit-status-description }} @@ -260,7 +250,7 @@ jobs: name: Post-Deploy Notifier if: always() needs: - - defaults # so we can access `inputs.root-sbd` that could have defaulted to `inputs.model` + - init # so we can access `inputs.root-sbd` that could have defaulted to `inputs.model` - deploy # so we can access deployment information such as versions used, status of deployments, etc. runs-on: ubuntu-latest permissions: @@ -311,11 +301,11 @@ jobs: # Creates a comment of the form: https://github.com/ACCESS-NRI/ACCESS-TEST/pull/15#issuecomment-2558675980 env: J2_MODEL: ${{ inputs.model }} - J2_PRERELEASE_VERSION: ${{ needs.defaults.outputs.prerelease-version }} - J2_ROOT_SBD: ${{ needs.defaults.outputs.root-sbd }} + J2_PRERELEASE_VERSION: ${{ needs.init.outputs.prerelease-version }} + J2_ROOT_SBD: ${{ needs.init.outputs.deployment-name }} J2_DEPLOYMENT_ERRORS: ${{ steps.deployment-rollup.outputs.errors }} J2_CONFIG_ERRORS: ${{ steps.config-rollup.outputs.errors }} - J2_HEAD_SHA: ${{ needs.defaults.outputs.head-sha }} + J2_HEAD_SHA: ${{ needs.init.outputs.head-sha }} J2_RUN_URL: ${{ env.RUN_URL }} J2_SPACK_MANIFEST_PATH: ${{ inputs.spack-manifest-path }} run: | @@ -336,9 +326,9 @@ jobs: PR_BODY_PATH: ./body.txt PR_BODY_PATH_UPDATED: ./updated.body.txt PRERELEASE_SECTION_REGEX: "^:rocket: .* :rocket:$" - PRERELEASE_SECTION: ":rocket: The latest prerelease `${{ needs.defaults.outputs.root-sbd }}/${{ needs.defaults.outputs.prerelease-version }}` at ${{ needs.defaults.outputs.head-sha }} is here: ${{ steps.comment.outputs.comment-link }} :rocket:" + PRERELEASE_SECTION: ":rocket: The latest prerelease `${{ needs.init.outputs.deployment-name }}/${{ needs.init.outputs.prerelease-version }}` at ${{ needs.init.outputs.head-sha }} is here: ${{ steps.comment.outputs.comment-link }} :rocket:" run: | - gh pr view ${{ inputs.pr }} --repo ${{ github.repository }} --json body --jq .body > ${{ env.PR_BODY_PATH }} + gh pr view ${{ env.PR_NUMBER }} --repo ${{ github.repository }} --json body --jq .body > ${{ env.PR_BODY_PATH }} # `awk` is a series of `CONDITION { ACTION }` pairs. No 'CONDITION' means 'TRUE', no '{ ACTION }' means 'print'. if grep -q '${{ env.PRERELEASE_SECTION_REGEX }}' ${{ env.PR_BODY_PATH }}; then # there is an existing prerelease section @@ -354,4 +344,4 @@ jobs: cat ${{ env.PR_BODY_PATH_UPDATED }} - gh pr edit ${{ inputs.pr }} --repo ${{ github.repository }} --body-file ${{ env.PR_BODY_PATH_UPDATED }} + gh pr edit ${{ env.PR_NUMBER }} --repo ${{ github.repository }} --body-file ${{ env.PR_BODY_PATH_UPDATED }} diff --git a/.github/workflows/create-deployment-spack.yml b/.github/workflows/create-deployment-spack.yml index d108e68e..e7a54cb3 100644 --- a/.github/workflows/create-deployment-spack.yml +++ b/.github/workflows/create-deployment-spack.yml @@ -11,11 +11,6 @@ on: type: string required: true description: A version of spack - spack-packages-version: - type: string - required: true - default: main - description: A version of ACCESS-NRI/spack-packages spack-config-version: type: string required: true @@ -33,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - run: | - echo '::notice::url = `${{ inputs.spack-git-url }}`, spack version = `${{ inputs.spack-version }}`, spack-packages version = `${{ inputs.spack-packages-version }}`, spack-config version = `${{ inputs.spack-config-version }}`, deployment location = `${{ inputs.deployment-location }}`' + echo '::notice::url = `${{ inputs.spack-git-url }}`, spack version = `${{ inputs.spack-version }}`, spack-config version = `${{ inputs.spack-config-version }}`, deployment location = `${{ inputs.deployment-location }}`' create-spack: name: Spack @@ -63,7 +58,6 @@ jobs: ssh ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' mkdir ${{ env.ROOT_VERSION_LOCATION }} || exit $? git -C ${{ env.ROOT_VERSION_LOCATION }} clone -c feature.manyFiles=true ${{ inputs.spack-git-url }} --branch ${{ inputs.spack-version }} --single-branch --depth=1 - git -C ${{ env.ROOT_VERSION_LOCATION }} clone https://github.com/ACCESS-NRI/spack-packages.git --branch ${{ inputs.spack-packages-version }} git -C ${{ env.ROOT_VERSION_LOCATION }} clone https://github.com/ACCESS-NRI/spack-config.git --branch ${{ inputs.spack-config-version }} ln -s -r -v ${{ env.ROOT_VERSION_LOCATION }}/spack-config/v${{ steps.strip.outputs.version-dir }}/${{ vars.DEPLOYMENT_TARGET }}/* ${{ env.ROOT_VERSION_LOCATION }}/spack/etc/spack/ mkdir ${{ env.ROOT_VERSION_LOCATION }}/release @@ -75,6 +69,6 @@ jobs: mkdir -p $TMPDIR/restricted/spack-stage setfacl --recursive -m \ - "g:vk83_w:r-X,g:ki32_mosrs:r-X,g:vk83_um:r-X,g::---,other::---,d:g:vk83_w:r-X,d:g:ki32_mosrs:r-X,d:g:vk83_um:r-X,d:g::---,d:other::---" \ + "g:vk83_w:r-X,g:ki32_mosrs:r-X,g:vk83_um:r-X,g:df42:r-X,g::---,o::---,d:g:vk83_w:r-X,d:g:ki32_mosrs:r-X,d:g:vk83_um:r-X,d:g:df42:r-X,d:g::---,d:o::---" \ ${{ env.ROOT_VERSION_LOCATION }}/restricted $TMPDIR/restricted EOT diff --git a/.github/workflows/deploy-1-setup.yml b/.github/workflows/deploy-1-setup.yml index fc245245..08a069dc 100644 --- a/.github/workflows/deploy-1-setup.yml +++ b/.github/workflows/deploy-1-setup.yml @@ -106,14 +106,22 @@ on: # value: ${{ jobs.check-config.outputs.spack-config-git-hash }} # description: | # Git hash of the 'access-nri/spack-config' repository at the specified version. - # spack-packages-version: - # value: ${{ jobs.check-config.outputs.spack-packages-version }} + # builtin-spack-packages-version: + # value: ${{ jobs.check-config.outputs.builtin-spack-packages-version }} # description: | - # Git ref of 'access-nri/spack-packages' that is used to reference custom packages in spack. - # spack-packages-git-hash: - # value: ${{ jobs.check-config.outputs.spack-packages-git-hash }} + # Git ref of 'spack/spack-packages' that is used to reference builtin packages in spack. + # builtin-spack-packages-git-hash: + # value: ${{ jobs.check-config.outputs.builtin-spack-packages-git-hash }} # description: | - # Git hash of the 'access-nri/spack-packages' repository at the specified version. + # Git hash of the 'spack/spack-packages' repository at the specified version. + # access-spack-packages-version: + # value: ${{ jobs.check-config.outputs.access-spack-packages-version }} + # description: | + # Git ref of 'access-nri/access-spack-packages' that is used to reference custom packages in spack. + # access-spack-packages-git-hash: + # value: ${{ jobs.check-config.outputs.access-spack-packages-git-hash }} + # description: | + # Git hash of the 'access-nri/access-spack-packages' repository at the specified version. # release-deployment-version: # value: ${{ jobs.check-spack-yaml.outputs.release }} # description: | @@ -146,79 +154,119 @@ jobs: check-config: name: '${{ inputs.deployment-target }}: Check Config' runs-on: ubuntu-latest + # A lot of outputs, versions for repositories are taken from different sources: + # Taken from central build-cd config/settings.json: spack (SHA), spack-config, builtin spack-packages + # Taken from caller config/versions.json: spack (branch), access-spack-packages + env: + # Paths prepended with actions/checkout path argument + mdr-config-versions-json-path: model/config/versions.json + mdr-config-packages-json-path: model/config/packages.json + build-cd-config-settings-json-path: build-cd/config/settings.json outputs: - spack-version: ${{ steps.spack.outputs.version }} - spack-git-hash: ${{ steps.spack.outputs.git-hash }} - spack-packages-version: ${{ steps.spack-packages.outputs.version }} - spack-packages-git-hash: ${{ steps.spack-packages.outputs.git-hash }} - spack-config-version: ${{ steps.spack-config.outputs.version }} - spack-config-git-hash: ${{ steps.spack-config.outputs.git-hash }} + spack-version: ${{ steps.config-versions-refs.outputs.spack-branch }} + spack-git-hash: ${{ steps.spack-sha.outputs.sha }} + + builtin-spack-packages-version: ${{ steps.config-settings-refs.outputs.builtin-spack-packages-ref }} + builtin-spack-packages-git-hash: ${{ steps.builtin-spack-packages-sha.outputs.sha }} + + access-spack-packages-version: ${{ steps.config-versions-refs.outputs.access-spack-packages-ref }} + access-spack-packages-git-hash: ${{ steps.access-spack-packages-sha.outputs.sha }} + + spack-config-version: ${{ steps.config-settings-refs.outputs.spack-config-ref }} + spack-config-git-hash: ${{ steps.spack-config-sha.outputs.sha }} + config-settings-failures: ${{ steps.settings.outputs.failures }} steps: - - name: Checkout ${{ github.repository }} Config + # Validate the caller repository's config files and versions + - name: '${{ github.repository }}: Checkout ${{ github.repository }} Config' uses: actions/checkout@v4 with: path: model ref: ${{ inputs.deployment-ref }} - - name: Validate ${{ github.repository }} config/versions.json + - name: '${{ github.repository }}: Validate config/versions.json' uses: access-nri/schema/.github/actions/validate-with-schema@main with: schema-version: ${{ inputs.config-versions-schema-version }} schema-location: au.org.access-nri/model/deployment/config/versions - data-location: ./model/config/versions.json + data-location: ${{ env.mdr-config-versions-json-path }} - - name: Validate ${{ github.repository }} config/packages.json + - name: '${{ github.repository }}: Validate config/packages.json' uses: access-nri/schema/.github/actions/validate-with-schema@main with: schema-version: ${{ inputs.config-packages-schema-version }} schema-location: au.org.access-nri/model/deployment/config/packages - data-location: ./model/config/packages.json + data-location: ${{ env.mdr-config-packages-json-path }} - - name: Validate spack-packages version - id: spack-packages - uses: access-nri/build-cd/.github/actions/validate-repo-version@v7 + - name: '${{ github.repository }}: Get repository refs from config/versions.json' + id: config-versions-refs + run: | + echo "spack-branch=$(jq --compact-output --raw-output '.spack' ${{ env.mdr-config-versions-json-path}})" >> $GITHUB_OUTPUT + echo "access-spack-packages-ref=$(jq --compact-output --raw-output '."access-spack-packages"' ${{ env.mdr-config-versions-json-path }})" >> $GITHUB_OUTPUT + + - name: '${{ github.repository }}: Validate spack branch' + id: spack-branch-check + uses: access-nri/actions/.github/actions/get-git-ref-info@main with: - repo-to-check: spack-packages - pr: ${{ inputs.deployment-ref }} + repository: ACCESS-NRI/spack + ref: releases/v${{ steps.config-versions-refs.outputs.spack-branch }} - - name: Validate spack version - id: spack - uses: access-nri/build-cd/.github/actions/validate-repo-version@v7 + - name: '${{ github.repository }}: Get access-spack-packages SHA' + id: access-spack-packages-sha + uses: access-nri/actions/.github/actions/get-git-ref-info@main with: - repo-to-check: spack - pr: ${{ inputs.deployment-ref }} + repository: ACCESS-NRI/access-spack-packages + ref: ${{ steps.config-versions-refs.outputs.access-spack-packages-ref }} - - name: Checkout build-cd Config + # Validate the build-cd repository's config/settings.json for the deployment target + - name: 'build-cd: Checkout build-cd Config' uses: actions/checkout@v4 with: repository: access-nri/build-cd - path: cd + path: build-cd - - name: Get spack-config version - id: spack-config - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: 'build-cd: Validate config/settings.json' + id: settings + uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v8 + with: + settings-path: ${{ env.build-cd-config-settings-json-path }} + target: ${{ inputs.deployment-target }} + + - name: 'build-cd: Get repository refs from config/settings.json' + id: config-settings-refs run: | - version=$(jq --compact-output --raw-output \ - --arg spack_version "${{ steps.spack.outputs.version }}" \ - '.deployment.${{ inputs.deployment-target }}.${{ inputs.deployment-type }}[$spack_version]."spack-config"' cd/config/settings.json + refs_for_deployment=$(jq \ + --arg spack_version "${{ steps.config-versions-refs.outputs.spack-branch }}" \ + '.deployment.${{ inputs.deployment-target }}.${{ inputs.deployment-type }}[$spack_version]' \ + ${{ env.build-cd-config-settings-json-path }} ) - echo $version - echo "version=$version" >> $GITHUB_OUTPUT - git_hash=$(gh api /repos/access-nri/spack-config/tags \ - | jq --compact-output --raw-output --arg v "$version" '.[] | select(.name == $v) | .commit.sha' - ) - echo $git_hash - echo "git-hash=$git_hash" >> $GITHUB_OUTPUT + echo "$refs_for_deployment" - - name: Validate build-cd config/settings.json - id: settings - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v7 + echo "spack-ref=$(jq --compact-output --raw-output '.spack' <<< "$refs_for_deployment")" >> $GITHUB_OUTPUT + echo "spack-config-ref=$(jq --compact-output --raw-output '."spack-config"' <<< "$refs_for_deployment")" >> $GITHUB_OUTPUT + echo "builtin-spack-packages-ref=$(jq --compact-output --raw-output '."builtin-spack-packages"' <<< "$refs_for_deployment")" >> $GITHUB_OUTPUT + + - name: 'build-cd: Get spack SHA' + id: spack-sha + uses: access-nri/actions/.github/actions/get-git-ref-info@main with: - settings-path: ./cd/config/settings.json - target: ${{ inputs.deployment-target }} + repository: ACCESS-NRI/spack + ref: ${{ steps.config-settings-refs.outputs.spack-ref }} + + - name: 'build-cd: Get spack-config SHA' + id: spack-config-sha + uses: access-nri/actions/.github/actions/get-git-ref-info@main + with: + repository: ACCESS-NRI/spack-config + ref: ${{ steps.config-settings-refs.outputs.spack-config-ref }} + + - name: 'build-cd: Get builtin spack-packages SHA' + id: builtin-spack-packages-sha + uses: access-nri/actions/.github/actions/get-git-ref-info@main + with: + repository: spack/spack-packages + ref: ${{ steps.config-settings-refs.outputs.builtin-spack-packages-ref }} check-spack-yaml: name: '${{ inputs.deployment-target }}: Check Spack Manifest' @@ -231,7 +279,7 @@ jobs: pull-requests: write outputs: # Release version of the deployment. Inferred if not given in inputs.deployment-version - release: ${{ steps.current.outputs.root-spec-ref }} + release: ${{ steps.current.outputs.deployment-version }} # Spack env name in form - spack-env-name: ${{ steps.get-env-name.outputs.env-name }} steps: @@ -250,21 +298,19 @@ jobs: - name: Get current (${{ inputs.deployment-ref }}) root spec version id: current - uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7 + uses: access-nri/build-cd/.github/actions/get-spack-manifest@v8 with: spack-manifest-path: ${{ inputs.spack-manifest-path }} - - name: Check root spec usage - if: env.IS_NOT_DRAFT == 'true' || inputs.deployment-type == 'Release' + - name: Disallow multi-spec Releases + if: inputs.deployment-type == 'Release' + # FIXME: Once the necessary infrastructure/guidance is in place, remove this step. See ACCESS-NRI/build-cd#344 run: | - if [[ "${{ steps.current.outputs.root-spec-ref }}" == "latest" ]]; then - echo "::error::The '@latest' version string is not allowed in non-draft PRs. It is reserved for a spack package that moves often and cannot be used as a future release tag" - exit 1 - fi + root_specs=$(yq '.spack.specs | select(.[] == "${{ steps.current.outputs.deployment-name }}*") | length' ${{ inputs.spack-manifest-path }}) - if [[ "${{ inputs.expected-root-spec-name }}" != "${{ steps.current.outputs.root-spec-name }}" ]]; then - echo "::error::The root spec in the spack manifest (${{ steps.current.outputs.root-spec-name }}) cannot be different from the expected root spec (${{ inputs.expected-root-spec-name }}) in non-Draft PRs." - exit 1 + if [ "$root_specs" -gt 1 ]; then + echo "::error::We don't yet support multiple root specs with the same package name. Contact the Model Release Team for information." + exit 1 fi - name: Checkout base (${{ inputs.prerelease-compare-ref }}) spack manifest @@ -289,7 +335,7 @@ jobs: - name: Get base root spec version id: base if: inputs.deployment-type != 'Release' && steps.checkout-base-spack.outcome != 'failure' - uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7 + uses: access-nri/build-cd/.github/actions/get-spack-manifest@v8 with: spack-manifest-path: ${{ inputs.spack-manifest-path }} @@ -300,7 +346,7 @@ jobs: if: inputs.deployment-type != 'Release' && steps.checkout-base-spack.outcome != 'failure' && env.IS_NOT_DRAFT == 'true' id: version-modified run: | - if [[ "${{ steps.base.outputs.root-spec-ref }}" == "${{ steps.current.outputs.root-spec-ref }}" ]]; then + if [[ "${{ steps.base.outputs.deployment-version }}" == "${{ steps.current.outputs.deployment-version }}" ]]; then echo "::warning::The version string in ${{ inputs.spack-manifest-path }} hasn't been modified in this PR, but needs to be before merging." exit 1 fi @@ -330,7 +376,7 @@ jobs: needs: - check-config # Verify configuration information is correct - check-spack-yaml # Verify spack manifest information is correct - uses: access-nri/build-cd/.github/workflows/deploy-2-start.yml@v7 + uses: access-nri/build-cd/.github/workflows/deploy-2-start.yml@v8 with: ref: ${{ inputs.deployment-ref }} version: ${{ inputs.deployment-version }} @@ -369,8 +415,10 @@ jobs: spack_git_hash: "${{ needs.check-config.outputs.spack-git-hash }}", spack_config_version: "${{ needs.check-config.outputs.spack-config-version }}", spack_config_git_hash: "${{ needs.check-config.outputs.spack-config-git-hash }}", - spack_packages_version: "${{ needs.check-config.outputs.spack-packages-version }}", - spack_packages_git_hash: "${{ needs.check-config.outputs.spack-packages-git-hash }}", + builtin_spack_packages_version: "${{ needs.check-config.outputs.builtin-spack-packages-version }}", + builtin_spack_packages_git_hash: "${{ needs.check-config.outputs.builtin-spack-packages-git-hash }}", + access_spack_packages_version: "${{ needs.check-config.outputs.access-spack-packages-version }}", + access_spack_packages_git_hash: "${{ needs.check-config.outputs.access-spack-packages-git-hash }}", ci_configuration_check_failure: ${{ needs.check-config.outputs.config-settings-failures != '' }}, release_deployment_version: "${{ needs.check-spack-yaml.outputs.release }}", spack_environment_name: "${{ needs.check-spack-yaml.outputs.spack-env-name }}", diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml index ff676978..837e5416 100644 --- a/.github/workflows/deploy-2-start.yml +++ b/.github/workflows/deploy-2-start.yml @@ -61,8 +61,6 @@ jobs: runs-on: ubuntu-latest environment: ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} outputs: - packages-version: ${{ steps.versions.outputs.packages }} - config-version: ${{ steps.versions.outputs.config }} spack-location: ${{ steps.location.outputs.spack }} modules-location: ${{ steps.location.outputs.modules }} root-spec-pkg-hash: ${{ steps.metadata.outputs.root-spec-pkg-hash }} @@ -77,29 +75,25 @@ jobs: uses: actions/checkout@v4 with: repository: access-nri/build-cd - path: cd + path: build-cd - name: Get Versions From config/versions.json id: versions run: | echo "spack=$(jq --compact-output --raw-output '.spack' ./config/versions.json)" >> $GITHUB_OUTPUT - echo "packages=$(jq --compact-output --raw-output '."spack-packages"' ./config/versions.json)" >> $GITHUB_OUTPUT + echo "packages=$(jq --compact-output --raw-output '."access-spack-packages"' ./config/versions.json)" >> $GITHUB_OUTPUT + echo "custom-scopes=$(jq --compact-output --raw-output '."custom-scopes" // [] | join(" ")' ./config/versions.json)" >> $GITHUB_OUTPUT - name: Get ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} Remote Paths id: path - uses: access-nri/build-cd/.github/actions/get-deploy-paths@v7 + uses: access-nri/build-cd/.github/actions/get-deploy-paths@v8 with: spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_LOCATION }} spack-version: ${{ steps.versions.outputs.spack }} - deployment-environment: ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} + deployment-target: ${{ inputs.deployment-target }} + deployment-type: ${{ inputs.deployment-type }} spack-environment: ${{ inputs.env-name }} - - name: Get manifest info - id: manifest - uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7 - with: - spack-manifest-path: ${{ inputs.spack-manifest-path }} - - name: Setup SSH id: ssh uses: access-nri/actions/.github/actions/setup-ssh@main @@ -116,13 +110,13 @@ jobs: cache: 'pip' - name: Install Injection Requirements - working-directory: cd + working-directory: build-cd run: python -m pip install -r scripts/spack_manifest/injection/requirements.txt - name: Spack Manifest - Get Provenance and Injection Packages id: config-packages run: | - packages_for_injection=$(jq -cr '.provenance + .injection | join(" ")' config/packages.json) + packages_for_injection=$(jq -cr '.provenance + .injection | join(",")' config/packages.json) packages_for_provenance=$(jq -cr '.provenance | join(" ")' config/packages.json) echo "Packages to be injected: $packages_for_injection" @@ -133,31 +127,31 @@ jobs: - name: Spack Manifest - Modules Injection # Injects relevant module projections and includes into the spack manifest so users don't have to - id: modules-injection - working-directory: cd + working-directory: build-cd run: | - original_root_spec_projection=$(yq '.spack.modules.default.tcl.projections."${{ steps.manifest.outputs.root-spec-name }}"' ../${{ inputs.spack-manifest-path }}) - if [[ "$original_root_spec_projection" != "null" ]]; then - echo "custom-projection=${original_root_spec_projection}" >> $GITHUB_OUTPUT - fi - python -m scripts.spack_manifest.injection.modules \ --manifest ../${{ inputs.spack-manifest-path }} \ --packages "${{ steps.config-packages.outputs.packages-for-injection }}" \ --output ../${{ inputs.spack-manifest-path }} + - name: Get SHA for access-spack-packages repository + id: packages-ref + uses: access-nri/actions/.github/actions/get-git-ref-info@main + with: + repository: ACCESS-NRI/access-spack-packages + ref: ${{ steps.versions.outputs.packages }} + - name: Spack Manifest - Prerelease Injection if: inputs.deployment-type == 'Prerelease' # Modifies the name of the prerelease modulefile to the pr-` style. - # Also removes the `@git.VERSION` specifier for Prereleases as it is a tag that doesn't yet exist. - # Also adds a repos section that points to the prereleases specific spack-packages repo. - working-directory: cd + # Also adds a repos section that points to the prereleases specific access-spack-packages repo. + working-directory: build-cd run: | python -m scripts.spack_manifest.injection.prerelease \ --manifest ../${{ inputs.spack-manifest-path }} \ --version ${{ inputs.version }} \ - ${{ steps.modules-injection.outputs.custom-projection != '' && format('--custom-root-projection {0}', steps.modules-injection.outputs.custom-projection ) || ''}} \ - --spack-packages-path ${{ steps.path.outputs.spack-packages }} \ + --spack-packages-path ${{ steps.path.outputs.spack-environment }}/access-spack-packages \ + --spack-packages-version-sha ${{ steps.packages-ref.outputs.sha }} \ --output ../${{ inputs.spack-manifest-path }} - name: Copy ${{ inputs.spack-manifest-path }} @@ -184,37 +178,11 @@ jobs: done EOT - - name: Update spack-packages repository - run: | - ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' - # Check that a suitable spack-packages location exists - if [ ! -d '${{ steps.path.outputs.spack-packages-root }}' ]; then - echo '::error::spack-packages does not exist in ${{ steps.path.outputs.root }} for ${{ inputs.deployment-target }} ${{ inputs.deployment-type}}' - exit 1 - fi - - # In the case of a Prerelease, we need to create a new instance of spack-packages per commit, so there is no concurrent access problems. - if [ ! -d '${{ steps.path.outputs.spack-packages }}' ]; then - echo 'Creating spack-packages instance for ${{ inputs.env-name }} under ${{ steps.path.outputs.spack-packages }}' - mkdir -p ${{ steps.path.outputs.spack-packages }} - git clone https://github.com/ACCESS-NRI/spack-packages.git '${{ steps.path.outputs.spack-packages }}' - fi - - cd ${{ steps.path.outputs.spack-packages }} || exit 1 - - git fetch --all --tags - - if git ls-remote --exit-code --heads origin ${{ steps.versions.outputs.packages }}; then # If the ref is a branch - echo "Checking out the branch at the latest commit" - git checkout -B ${{ steps.versions.outputs.packages }} origin/${{ steps.versions.outputs.packages }} - else # checkout the tag/commit - echo "Checking out the tag/commit" - git checkout ${{ steps.versions.outputs.packages }} - fi - EOT - - name: Deploy to ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} id: deploy + env: + SCOPES: ${{ steps.versions.outputs.custom-scopes }} + SCOPES_PATH: ${{ steps.path.outputs.spack-config }}/custom/cd # ssh into deployment environment, create and activate the env, install the spack manifest. run: | ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' @@ -231,11 +199,17 @@ jobs: # Enable spack . ${{ steps.path.outputs.spack-config }}/spack-enable.bash - # Create environment and build model - spack env create ${{ inputs.env-name }} ${{ vars.SPACK_YAML_LOCATION }}/${{ inputs.expected-root-spec-name }}-${{ github.run_id }}.spack.yaml - spack env activate ${{ inputs.env-name }} - spack --debug install --fail-fast --fresh ${{ vars.SPACK_INSTALL_ADDITIONAL_ARGS }} || exit $? - spack module tcl refresh -y + # Create and activate environment + spack env activate ${{ inputs.env-name }} --create --envfile ${{ vars.SPACK_YAML_LOCATION }}/${{ inputs.expected-root-spec-name }}-${{ github.run_id }}.spack.yaml + + # Finally, install the spack manifest + spack --debug \ + $(for s in ${{ env.SCOPES }}; do echo -n "--config-scope=${{ env.SCOPES_PATH }}/$s "; done) \ + install --fail-fast --fresh ${{ vars.SPACK_INSTALL_ADDITIONAL_ARGS }} || exit $? + + spack \ + $(for s in ${{ env.SCOPES }}; do echo -n "--config-scope=${{ env.SCOPES_PATH }}/$s "; done) \ + module tcl refresh -y EOT - name: Move spack logs @@ -283,7 +257,7 @@ jobs: hash=$(spack find --format "{^${pkg}.hash}" /$root_spec_hash) version=$(spack find --format '{version}' /$hash) location=$(spack find --format '{prefix}' /$hash) - pkg_repo_url=$(spack python -c "import spack.spec; print(spack.spec.Spec('$pkg').package_class.git)") + pkg_repo_url=$(spack python -c "print(spack.repo.PATH.get_pkg_class('$pkg').git)") # We need to create a tmp output file as jq doesn't support in-place read/write jq \ diff --git a/.github/workflows/settings-1-update.yml b/.github/workflows/settings-1-update.yml index ba8fef01..1ee5616e 100644 --- a/.github/workflows/settings-1-update.yml +++ b/.github/workflows/settings-1-update.yml @@ -53,7 +53,7 @@ jobs: - name: Validate Deployment Settings id: validate - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v7 + uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v8 with: settings-path: ${{ env.CONFIG_SETTINGS_PATH }} target: ${{ matrix.target }} @@ -146,7 +146,7 @@ jobs: # - deployment-environment: Gadi # type: Prerelease # etc ... - uses: access-nri/build-cd/.github/workflows/settings-2-deploy.yml@v7 + uses: access-nri/build-cd/.github/workflows/settings-2-deploy.yml@v8 with: deployment-environment: ${{ matrix.update.deployment-environment }} spack-type: ${{ matrix.update.type }} diff --git a/.github/workflows/settings-2-deploy.yml b/.github/workflows/settings-2-deploy.yml index 5cc77972..39989a43 100644 --- a/.github/workflows/settings-2-deploy.yml +++ b/.github/workflows/settings-2-deploy.yml @@ -55,6 +55,23 @@ jobs: echo "$updates" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT + - name: Setup builtin spack-packages updates + id: builtin-spack-packages + # Notably, only the spack instances that are >=1.0 and have builtin-spack-packages entries will produce any output here + run: | + updates=$(jq --compact-output --raw-output \ + --arg env "${{ inputs.deployment-environment }}" \ + --arg type "${{ inputs.spack-type }}" \ + '.deployment[$env][$type] | to_entries[] | select(.value."builtin-spack-packages" != null) | "\(.key) \(.value."builtin-spack-packages")"' \ + ${{ env.CONFIG_SETTINGS_PATH }} + ) + + echo "$updates" + # For multiline output, use a heredoc. See https://github.com/orgs/community/discussions/116619#discussioncomment-8994849 + echo "updates<> $GITHUB_OUTPUT + echo "$updates" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Setup SSH id: ssh uses: access-nri/actions/.github/actions/setup-ssh@main @@ -119,10 +136,46 @@ jobs: if [ $? -ne 0 ]; then echo "::error::Error: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version spack-config failed checkout from $current_head_commit to $new_commit" else + ln -s -r -v -f \ + ${{ secrets.SPACK_INSTALLS_ROOT_LOCATION }}/$version/spack-config/$version/* \ + ${{ secrets.SPACK_INSTALLS_ROOT_LOCATION }}/$version/spack/etc/spack/ + echo "::notice::Changed: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version spack-config changed from $current_head_commit to $new_commit" fi + else echo "::notice::Unchanged: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version spack-config left at $current_head_commit" fi done <<< "${{ steps.spack-config.outputs.updates }}" EOT + + - name: Update builtin spack-packages + # This will only run for instances that have builtin-spack-packages defined in their entries (i.e. instances at spack >=1.0) + continue-on-error: true + run: | + ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + set +e + + while read -ra update; do + version=${update[0]} + new_ref=${update[1]} + + . ${{ secrets.SPACK_INSTALLS_ROOT_LOCATION }}/$version/spack-config/spack-enable.bash + + builtin_repo_path=$(spack location --repo builtin) + + current_head_commit=$(git -C $builtin_repo_path rev-parse HEAD) + new_commit=$(git -C $builtin_repo_path rev-parse "$new_ref") + + if [[ "$current_head_commit" != "$new_commit" ]]; then + + spack repo update builtin --commit $new_commit + + if [ $? -ne 0 ]; then + echo "::error::Error: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version builtin-spack-packages failed checkout from $current_head_commit to $new_commit" + else + echo "::notice::Changed: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version builtin-spack-packages changed from $current_head_commit to $new_commit" + fi + else + echo "::notice::Unchanged: ${{ inputs.deployment-environment }} ${{ inputs.spack-type }} $version builtin-spack-packages left at $current_head_commit" + fi diff --git a/.github/workflows/undeploy-1-start.yml b/.github/workflows/undeploy-1-start.yml index ba60108f..69e0aea9 100644 --- a/.github/workflows/undeploy-1-start.yml +++ b/.github/workflows/undeploy-1-start.yml @@ -48,11 +48,12 @@ jobs: - name: Get ${{ inputs.target }} Remote Paths id: path - uses: access-nri/build-cd/.github/actions/get-deploy-paths@v7 + uses: access-nri/build-cd/.github/actions/get-deploy-paths@v8 with: spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_LOCATION }} spack-version: ${{ steps.versions.outputs.spack }} - deployment-environment: ${{ inputs.target }} ${{ inputs.type }} + deployment-target: ${{ inputs.target }} + deployment-type: ${{ inputs.type }} spack-environment: ${{ inputs.version-pattern }} - name: Setup SSH @@ -88,17 +89,6 @@ jobs: for scope in $scopes; do spack --config-scope $scope gc --except-any-environment --yes-to-all done - - EOT - - - name: Undeploy spack-packages - # ssh into deployment environment, remove all the unneeded spack-packages installations - run: | - ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' - find "${{ steps.path.outputs.spack-packages-root }}" \ - -mindepth 1 -maxdepth 1 \ - -type d -name "${{ inputs.version-pattern }}" \ - -exec rm -rfv {} + EOT - name: Undeploy Status Notifier diff --git a/config/README.md b/config/README.md index 91e4354c..d600032a 100644 --- a/config/README.md +++ b/config/README.md @@ -15,3 +15,8 @@ This means that one should not modify the repositories versions on the HPC targe ## `settings.schema.json` This schema enforces the structure shown in `settings.json`. + +Note: + +- For deployments using spack version keys >= `1.0` (for example `"1.0"`, `"1.1"`, `"2.0"`), each versioned entry under `deployment///` must include the field `builtin-spack-packages` in addition to `spack` and `spack-config`. +- For version keys < `1.0` (for example `"0.22"`), only `spack` and `spack-config` are required. This is because historically `spack`s builtin `spack-packages` repository was versioned as part of the `spack` repository version. diff --git a/config/settings.json b/config/settings.json index ce416990..7c06f99b 100644 --- a/config/settings.json +++ b/config/settings.json @@ -8,14 +8,24 @@ "spack-config": "2024.03.22" }, "0.22": { - "spack": "559a4290961f1262a3a74b138223ebcb376f0a03", - "spack-config": "2025.08.000" + "spack": "422a81b3d75f3f119e7c06234418763803a4f070", + "spack-config": "2026.02.000" + }, + "1.1": { + "spack": "89b3f0baae13477ee5384cd30ebf512489a69890", + "builtin-spack-packages": "8ed34337e15d759890b68682d704aa55dd243537", + "spack-config": "2026.02.001" } }, "Prerelease": { "0.22": { - "spack": "559a4290961f1262a3a74b138223ebcb376f0a03", - "spack-config": "2025.08.000" + "spack": "422a81b3d75f3f119e7c06234418763803a4f070", + "spack-config": "2026.02.000" + }, + "1.1": { + "spack": "89b3f0baae13477ee5384cd30ebf512489a69890", + "builtin-spack-packages": "8ed34337e15d759890b68682d704aa55dd243537", + "spack-config": "2026.02.001" } } } diff --git a/config/settings.schema.json b/config/settings.schema.json index a3379ee5..0cbe15fd 100644 --- a/config/settings.schema.json +++ b/config/settings.schema.json @@ -16,7 +16,7 @@ "Release": { "type": "object", "patternProperties": { - "^.+$": { + "^0\\.\\d+$": { "type": "object", "properties": { "spack": { @@ -26,16 +26,32 @@ "type": "string" } }, - "additionalProperties": true, + "additionalProperties": false, "required": ["spack", "spack-config"] + }, + "^[1-9]\\d*\\.\\d+$": { + "type": "object", + "properties": { + "spack": { + "type": "string" + }, + "builtin-spack-packages": { + "type": "string" + }, + "spack-config": { + "type": "string" + } + }, + "additionalProperties": false, + "required": ["spack", "builtin-spack-packages", "spack-config"] } }, - "additionalProperties": true + "additionalProperties": false }, "Prerelease": { "type": "object", "patternProperties": { - "^.+$": { + "^0\\.\\d+$": { "type": "object", "properties": { "spack": { @@ -45,11 +61,27 @@ "type": "string" } }, - "additionalProperties": true, + "additionalProperties": false, "required": ["spack", "spack-config"] + }, + "^[1-9]\\d*\\.\\d+$": { + "type": "object", + "properties": { + "spack": { + "type": "string" + }, + "builtin-spack-packages": { + "type": "string" + }, + "spack-config": { + "type": "string" + } + }, + "additionalProperties": false, + "required": ["spack", "builtin-spack-packages", "spack-config"] } }, - "additionalProperties": true + "additionalProperties": false } }, "additionalProperties": true, diff --git a/scripts/jinja_template/render_deployment_info.py b/scripts/jinja_template/render_deployment_info.py index abd550a1..211445b7 100644 --- a/scripts/jinja_template/render_deployment_info.py +++ b/scripts/jinja_template/render_deployment_info.py @@ -29,8 +29,13 @@ def _build_deployments_context_from_folder( "result": data["deployment_result"], "release_version": data["release_deployment_version"], "spack_version": data["spack_version"], + "spack_git_hash": data["spack_git_hash"], "spack_config_version": data["spack_config_version"], - "spack_packages_version": data["spack_packages_version"], + "spack_config_git_hash": data["spack_config_git_hash"], + "builtin_spack_packages_version": data["builtin_spack_packages_version"], + "builtin_spack_packages_git_hash": data["builtin_spack_packages_git_hash"], + "access_spack_packages_version": data["access_spack_packages_version"], + "access_spack_packages_git_hash": data["access_spack_packages_git_hash"], "modules_location": data["deployment_modules_location"], "spack_location": data["deployment_spack_location"], } diff --git a/scripts/jinja_template/templates/deployment-comment-body.md.j2 b/scripts/jinja_template/templates/deployment-comment-body.md.j2 index 8541b2ca..25cbae0c 100644 --- a/scripts/jinja_template/templates/deployment-comment-body.md.j2 +++ b/scripts/jinja_template/templates/deployment-comment-body.md.j2 @@ -31,11 +31,12 @@ Due to inode-saving measures, one will have to manually untar the environment me Configuration Information This Prerelease is deployed using: -* `access-nri/spack` on branch [`{{ deployment.spack_version }}`](https://github.com/ACCESS-NRI/spack/tree/releases/v{{ deployment.spack_version }}) -* `access-nri/spack-packages` version [`{{ deployment.spack_packages_version }}`](https://github.com/ACCESS-NRI/spack-packages/releases/tag/{{ deployment.spack_packages_version }}) -* `access-nri/spack-config` version [`{{ deployment.spack_config_version }}`](https://github.com/ACCESS-NRI/spack-config/releases/tag/{{ deployment.spack_config_version }}) +* `access-nri/spack` on branch [`{{ deployment.spack_version }}`](https://github.com/ACCESS-NRI/spack/tree/{{ deployment.spack_git_hash }}) +* `spack/spack-packages` version [`{{ deployment.builtin_spack_packages_version }}`](https://github.com/spack/spack-packages/tree/{{ deployment.builtin_spack_packages_git_hash }}) +* `access-nri/access-spack-packages` version [`{{ deployment.access_spack_packages_version }}`](https://github.com/ACCESS-NRI/access-spack-packages/tree/{{ deployment.access_spack_packages_git_hash }}) +* `access-nri/spack-config` version [`{{ deployment.spack_config_version }}`](https://github.com/ACCESS-NRI/spack-config/tree/{{ deployment.spack_config_git_hash }}) -If the above was not what was expected, commit changes to `config/versions.json` in this PR. +If the above was not what was expected, commit changes to `config/versions.json` in this PR, or propose changes to `build-cd`s `config/settings.json`. diff --git a/scripts/jinja_template/templates/release-body.md.j2 b/scripts/jinja_template/templates/release-body.md.j2 index cd130e88..cf6b7364 100644 --- a/scripts/jinja_template/templates/release-body.md.j2 +++ b/scripts/jinja_template/templates/release-body.md.j2 @@ -4,7 +4,7 @@ This is an official release of `{{ model }}` `{{ version }}` to the HPC(s) {% fo {% for deployment in deployments %} ### `{{ deployment.name }}` -Deployed using [spack {{ deployment.spack_version }}](https://github.com/ACCESS-NRI/spack/tree/releases/{{ deployment.spack_version }}), [spack-packages {{ deployment.spack_packages_version }}](https://github.com/ACCESS-NRI/spack-packages/releases/tag/{{ deployment.spack_packages_version }}) and [spack-config {{ deployment.spack_config_version }}](https://github.com/ACCESS-NRI/spack-config/releases/tag/{{ deployment.spack_config_version }}). +Deployed using [spack {{ deployment.spack_version }}](https://github.com/ACCESS-NRI/spack/tree/{{ deployment.spack_git_hash }}), [spack/spack-packages {{ deployment.builtin_spack_packages_version }}](https://github.com/spack/spack-packages/tree/{{ deployment.builtin_spack_packages_git_hash }}), [ACCESS-NRI/access-spack-packages {{ deployment.access_spack_packages_version }}](https://github.com/ACCESS-NRI/access-spack-packages/tree/{{ deployment.access_spack_packages_git_hash }}) and [spack-config {{ deployment.spack_config_version }}](https://github.com/ACCESS-NRI/spack-config/tree/{{ deployment.spack_config_git_hash }}). Deployed as a module accessible using: diff --git a/scripts/release_provenance/tracking_services_data.py b/scripts/release_provenance/tracking_services_data.py index 1ac8cdbb..594986e7 100644 --- a/scripts/release_provenance/tracking_services_data.py +++ b/scripts/release_provenance/tracking_services_data.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any -TRACKING_SERVICES_JSON_SCHEMA_VERSION = "1-0-0" +TRACKING_SERVICES_JSON_SCHEMA_VERSION = "2-0-0" TRACKING_SERVICES_JSON_SCHEMA_URL = f"https://raw.githubusercontent.com/ACCESS-NRI/schema/main/au.org.access-nri/tracking_services/release_provenance/telemetry/{TRACKING_SERVICES_JSON_SCHEMA_VERSION}.json" @@ -121,8 +121,10 @@ def _format_telemetry_of_deployment_target( "deployment_target.spack_git_hash": outputs["spack_git_hash"], "deployment_target.spack_config_version": outputs["spack_config_version"], "deployment_target.spack_config_git_hash": outputs["spack_config_git_hash"], - "deployment_target.spack_packages_version": outputs["spack_packages_version"], - "deployment_target.spack_packages_git_hash": outputs["spack_packages_git_hash"], + "deployment_target.builtin_spack_packages_version": outputs["builtin_spack_packages_version"], + "deployment_target.builtin_spack_packages_git_hash": outputs["builtin_spack_packages_git_hash"], + "deployment_target.access_spack_packages_version": outputs["access_spack_packages_version"], + "deployment_target.access_spack_packages_git_hash": outputs["access_spack_packages_git_hash"], "deployment_target.module_use_location": outputs["deployment_modules_location"], "spack_model.name": root_spec, diff --git a/scripts/spack_manifest/getter.py b/scripts/spack_manifest/getter.py index 9a8ae583..71365897 100644 --- a/scripts/spack_manifest/getter.py +++ b/scripts/spack_manifest/getter.py @@ -15,6 +15,107 @@ class NoSectionComponentError(Exception): pass +class Specs: + def __init__(self, manifest: dict[str, Any]): + self.manifest = manifest + + self.specs: list[str] = self._get_specs_from_manifest_or_raise() + + def _get_specs_from_manifest_or_raise(self) -> list[str]: + defs: list[dict[str, Any]] = self.manifest.get("spack", {}).get( + "definitions", [] + ) + specs: list[str] = self.manifest.get("spack", {}).get("specs", []) + + # It's either in the multi-target format or the single target format, we just need to find which + # The multi-target format is of the form: + # spack: + # definitions: + # - ROOT_PACKAGE: [access-om2] + # # ... + # FIXME: Multi-target-formatted specs only have the first one picked up. See ACCESS-NRI/build-cd#343 + root_package_def = next( + (d["ROOT_PACKAGE"] for d in defs if "ROOT_PACKAGE" in d), [] + ) + if root_package_def != []: + return [root_package_def[0]] + elif len(specs) != 0: + return specs + else: + raise NoSectionError( + "No specs defined in the manifest spack.specs section for a single-target manifest." + ) + + @classmethod + def from_file(cls, manifest_path: str) -> "RootSpec": + from yaml import safe_load + + with open(manifest_path, "r") as file: + manifest = safe_load(file) + + return cls(manifest) + + def get_specs(self) -> list[str]: + return self.specs + + def get_specs_with_name(self, name: str) -> list[str]: + return [s for s in self.specs if s.startswith(name)] + +class ReservedDefinitions: + def __init__(self, manifest: dict[str, Any]): + self.manifest: dict[str, Any] = manifest + + self.reserved_definitions: list[dict[str, Any]] = ( + self._get_reserved_definitions_from_manifest_or_raise() + ) + + def _get_reserved_definitions_from_manifest_or_raise(self) -> dict[str, Any]: + definitions: list[dict[str, Any]] = self.manifest.get("spack", {}).get( + "definitions", [] + ) + + if definitions == []: + raise NoSectionError( + f"spack.definitions section not found in the manifest." + ) + + # Turn something with the spack-specific structure: + # {'definitions': [ + # {'_name': ['access-om2']}, + # {'_version': ['2025.02.100']}, + # {'something': ['else']} + # ]} + # Into a much easier to parse: + # {'name': 'access-om2', 'version': '2025.02.100'} + # Stripping out non-reserved definitions and unneeded single-element lists + reserved_definitions: dict[str, Any] = {} + for definition in definitions: + if len(definition) > 0: + reserved_name, reserved_value_list = list(definition.items())[0] + if reserved_name.startswith("_") and len(reserved_value_list) > 0: + reserved_name_no_underscore = reserved_name.lstrip("_") + # In future if we want to handle other reserved defs as lists, we can add a case statement here + reserved_definitions[reserved_name_no_underscore] = reserved_value_list[0] + + return reserved_definitions + + @classmethod + def from_file(cls, manifest_path: str) -> "ReservedDefinitions": + from yaml import safe_load + + with open(manifest_path, "r") as file: + manifest = safe_load(file) + + return cls(manifest) + + def get(self, definition: str) -> str: + if definition not in self.reserved_definitions: + raise NoSectionComponentError( + f"Reserved definition '{definition}' not found in the manifest spack.definitions section." + ) + + return self.reserved_definitions[definition] + class RootSpec: def __init__(self, manifest: dict[str, Any]): @@ -23,7 +124,9 @@ def __init__(self, manifest: dict[str, Any]): self.root_spec: str = self._get_root_spec_from_manifest_or_raise() def _get_root_spec_from_manifest_or_raise(self) -> str: - defs: list[dict[str, Any]] = self.manifest.get("spack", {}).get("definitions", []) + defs: list[dict[str, Any]] = self.manifest.get("spack", {}).get( + "definitions", [] + ) specs: list[str] = self.manifest.get("spack", {}).get("specs", []) # It's either in the multi-target format or the single target format, we just need to find which @@ -190,7 +293,6 @@ def get_package_full_version_requirement(self, name: str) -> str: return requirements[0] - def get_package_requirements(self, name: str) -> list[str]: if name not in self.packages: raise NoSectionComponentError( @@ -269,3 +371,13 @@ def get(self) -> dict[str, str]: .get("tcl", {}) .get("projections", {}) ) + + def get_projection_with_name(self, name: str) -> str | None: + return ( + self.manifest.get("spack", {}) + .get("modules", {}) + .get("default", {}) + .get("tcl", {}) + .get("projections", {}) + .get(name, None) + ) diff --git a/scripts/spack_manifest/injection/modules.py b/scripts/spack_manifest/injection/modules.py index a7f95c08..c2665be7 100644 --- a/scripts/spack_manifest/injection/modules.py +++ b/scripts/spack_manifest/injection/modules.py @@ -1,14 +1,24 @@ import argparse import yaml import sys +import re from typing import Any from scripts.spack_manifest.getter import ( - RootSpec, + ReservedDefinitions, Packages, Includes, Projections, + Specs ) +from scripts.spack_manifest.injection.yaml_representer import ( + YamlExplicitFlowStyleSequence, + yaml_explicit_flow_style_sequence_representer, + enforce_explicit_flow_style_definitions +) + +# We represent reserved definitions as in flow-style sequences (eg. `[a]` rather than `- a`), so it is more compact. +yaml.add_representer(YamlExplicitFlowStyleSequence, yaml_explicit_flow_style_sequence_representer) ################## # Main functions # @@ -19,25 +29,30 @@ def main(): # Get inputs args = parse_args(sys.argv[1:]) - packages: set[str] = set(args.packages.split()) + packages: set[str] = set(args.packages.split(",") if args.packages != '' else []) with open(args.manifest, "r") as file: manifest: dict[str, Any] = yaml.safe_load(file) # Inject manifest with projections and includes - root_spec_name: str = RootSpec(manifest).get_name() + + deployment_name: str = ReservedDefinitions(manifest).get("name") manifest_with_projections: dict[str, Any] = inject_projections( - manifest=manifest, root_spec=root_spec_name, packages=packages + manifest=manifest, root_spec=deployment_name, packages=packages ) manifest_with_projections_and_includes: dict[str, Any] = inject_includes( - manifest=manifest_with_projections, root_spec=root_spec_name, packages=packages + manifest=manifest_with_projections, root_spec=deployment_name, packages=packages + ) + + finalized_manifest: dict[str, Any] = enforce_explicit_flow_style_definitions( + manifest_with_projections_and_includes ) # Output the modified manifest dumped_manifest: str = yaml.dump( - manifest_with_projections_and_includes, + finalized_manifest, default_flow_style=False, sort_keys=False, ) @@ -48,7 +63,6 @@ def main(): with open(args.output, "w") as output_file: output_file.write(dumped_manifest) - def inject_projections( manifest: str, root_spec: str, packages: set[str] ) -> dict[str, Any]: @@ -72,10 +86,9 @@ def inject_projections( # To start with, add the projections that are already defined in the manifest new_projections: dict[str, str] = dict(defined_projections_dict) - if root_spec not in defined_projections: - new_projections.update( - generate_projection_for_root_spec_or_raise(manifest, root_spec) - ) + new_projections.update( + generate_projection_for_root_spec_or_raise(manifest, root_spec, defined_projections_dict.get(root_spec, '')) + ) for projection in projections_to_generate: new_projections.update( @@ -122,24 +135,51 @@ def inject_includes( def generate_projection_for_root_spec_or_raise( - manifest: dict[str, Any], root_spec_name: str + manifest: dict[str, Any], + root_spec_name: str, + root_spec_projection: str = '' ) -> dict[str, str]: - root_spec_getter = RootSpec(manifest) + reserved_definitions_getter = ReservedDefinitions(manifest) + specs_getter = Specs(manifest) - root_spec_name_from_definition: str = root_spec_getter.get_name() - version = root_spec_getter.get_ref() - - if root_spec_name_from_definition != root_spec_name: - raise ValueError( - f"Expected root spec name '{root_spec_name}' does not match the name in the root spec definition '{root_spec_name_from_definition}'. The --root-spec needs to be defined the same as the actual root spec." - ) + version = reserved_definitions_getter.get("version") print( - f"Extracted version '{version}' from root spec definition '{root_spec_getter.get()}'" + f"Extracted version '{version}' from _version definition'" ) - # We don't add a hash to the root spec projection, as it is a unique deployment - return {root_spec_name: f"{{name}}/{version}"} + # Essentially we're looking to infix the version between {name} and what comes after, if there exists a projection. + module_projection_pattern = re.compile( + r""" + ^(.*\{name\}[^\/]*?) # First group - Either '{name}' (most common) or a prefix like 'system-tools/{name}' (for SDRs) + (?:\/(.+))?$ # Optional second group - Rest of the projection after '{name}/', if given. Used for variants/descriptors in modulefile names + """, + re.VERBOSE + ) + + projection_components: re.Match | None = re.match(module_projection_pattern, root_spec_projection) + + # If there is a root spec projection and it is well-formed, we need to infix the version + if projection_components: + # We have a custom projection defined for the root spec, so we need to respect that + projection_groups: tuple[str] = projection_components.groups() + + updated_version: str = f"{projection_groups[0]}/{version}" + (f"/{projection_groups[1]}" if projection_groups[1] else "") + + print(f"Root spec already had a projection ({root_spec_projection}) so we infix the the version ({version}) to give: {updated_version}") + + return {root_spec_name: updated_version} + + # Otherwise, we need to construct a projection from scratch + root_specs_in_speclist = len(specs_getter.get_specs_with_name(root_spec_name)) + + if root_specs_in_speclist == 0: + raise ValueError(f"No specs with name {root_spec_name} in speclist") + elif root_specs_in_speclist == 1: + return {root_spec_name: f"{{name}}/{version}"} + else: + # If there are multiple of the same root spec (for example, different variants housed under the same environment), we need to demarcate the modulefile with a spack package hash + return {root_spec_name: f"{{name}}/{version}/{{hash:7}}"} def generate_projection_for_package_or_raise( @@ -184,7 +224,7 @@ def parse_args(args: list[str]) -> argparse.Namespace: "--packages", type=str, required=True, - help="List of space-separated packages (excluding the root spec) to be considered for projection injection", + help="List of comma-separated packages (excluding the root spec) to be considered for projection injection", ) parser.add_argument( @@ -196,12 +236,6 @@ def parse_args(args: list[str]) -> argparse.Namespace: parsed_args = parser.parse_args(args) - # Verifying that --packages are space-separated, which is a bit different from the usual comma-separated lists - if "," in parsed_args.packages: - raise ValueError( - "The --packages argument must be a space-separated list of package names." - ) - return parsed_args diff --git a/scripts/spack_manifest/injection/prerelease.py b/scripts/spack_manifest/injection/prerelease.py index 539d002b..4cffbadb 100644 --- a/scripts/spack_manifest/injection/prerelease.py +++ b/scripts/spack_manifest/injection/prerelease.py @@ -7,58 +7,49 @@ from copy import deepcopy from scripts.spack_manifest.getter import ( - RootSpec, + ReservedDefinitions, Projections, + Specs +) +from scripts.spack_manifest.injection.yaml_representer import ( + YamlExplicitFlowStyleSequence, + YamlExplicitQuotedString, + yaml_explicit_flow_style_sequence_representer, + yaml_explicit_quoted_string_representer, + enforce_explicit_flow_style_definitions ) -# PyYaml by default dumps unquoted strings if they look unambiguous, and quoted strings otherwise. -# PyYaml dumps '{name}/prX-Y' as a quoted str as it has '{' at the front and causes ambiguity -# But 'ROOT_SPEC/.dependencies/prX-Y/VERSION-{hash:7}' is dumped as an unquoted str as it is unambiguous -# So we need to wrap projections in a custom class that forces PyYaml to dump them as quoted strings. -class YamlExplicitQuotedString(str): - pass - - -def yaml_explicit_quoted_string_representer(dumper, data): - """ - Custom representer for YAML to ensure that some strings are quoted explicitly. - This is necessary for strings that are used as projections in spack manifests. - """ - return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="'") - - +# The yaml representer sometimes dumps ambiguous strings in the case of projections like `{name}/...` as unquoted strings, +# which is not handled by spack very well. yaml.add_representer(YamlExplicitQuotedString, yaml_explicit_quoted_string_representer) -### Actual methods begin here ### +# We represent reserved definitions as in flow-style sequences (eg. `[a]` rather than `- a`), so it is more compact. +yaml.add_representer(YamlExplicitFlowStyleSequence, yaml_explicit_flow_style_sequence_representer) +### Actual methods begin here ### def inject_prerelease_information( manifest_path: str, version: str, - custom_root_projection: str | None = None, - keep_root_spec_intact: bool = False, spack_packages_path: str | None = None, + spack_packages_version_sha: str | None = None, ) -> str: # In comparison to the projections script, this returns a string rather than a dict because we need to # add spack-specific, non-standard 'repo::' sections, which the yaml dumper does not support. with open(manifest_path, "r") as manifest_file: manifest: dict[str, Any] = yaml.safe_load(manifest_file) - root_spec_from_manifest = RootSpec(manifest) - root_spec_name = root_spec_from_manifest.get_name() + reserved_definitions_from_manifest = ReservedDefinitions(manifest) + root_spec_name = reserved_definitions_from_manifest.get("name") updated_manifest: dict[str, Any] = deepcopy(manifest) - # Remove @git.VERSION information from the root spec, since it will be a tag that does not yet exist for prereleases - # This does not include versions of the form @VERSION, which are the hallmark of software deployment repositories, - # or builds that explicitly ask to keep_root_spec_intact. - if not keep_root_spec_intact: - updated_manifest = remove_potential_root_spec_git_version(manifest) - - # We want the root spec projection to be of the form {name}/prX-Y + # We want the root spec projection to be of the form {name}/prX-Y for single specs, and + # {name}/prX-Y/DEMARCATOR for multiple specs, so we don't have modulefile clashes. + # The DEMARCATOR can be a custom projection, or {hash:7} if not supplied. updated_manifest = update_root_spec_projection_version( - updated_manifest, root_spec_name, version, custom_root_projection + updated_manifest, root_spec_name, version ) # We want all other projections to be of the form {name}/prX-Y/VERSION @@ -66,15 +57,19 @@ def inject_prerelease_information( updated_manifest, root_spec_name, version ) + if spack_packages_path: + # Add the 'repo:' section for prerelease spack packages if provided + updated_manifest = add_prerelease_repos_section( + updated_manifest, spack_packages_path, spack_packages_version_sha + ) + + updated_manifest = enforce_explicit_flow_style_definitions(updated_manifest) + # Dump the current dict, and add the non-standard 'repo::' section manifest_str: str = yaml.dump( updated_manifest, default_flow_style=False, sort_keys=False ) - if spack_packages_path: - # Add the 'repo::' section for prerelease spack packages if provided - manifest_str = add_prerelease_repos_section(manifest_str, spack_packages_path) - return manifest_str @@ -107,55 +102,60 @@ def add_namespace_to_other_projection_versions( return manifest -def remove_potential_root_spec_git_version(manifest: dict[str, Any]) -> dict[str, Any]: - """ - Remove the version information from the root spec in the manifest. - This is necessary for prerelease deployments where the version may not yet exist. - """ - root_spec_from_manifest = RootSpec(manifest) - name = root_spec_from_manifest.get_name() - constraints = root_spec_from_manifest.get_non_version_constraints() - - if root_spec_from_manifest.has_git_ref(): - # Remove the @git version and then add later contraints back - manifest["spack"]["specs"][0] = f"{name} {constraints}".strip() - else: - print( - f"The root spec '{name}' does not have a git ref, so no changes are made." - ) - - return manifest - - def update_root_spec_projection_version( - manifest: dict[str, Any], root_spec_name: str, root_spec_version: str, custom_root_projection: str | None = None + manifest: dict[str, Any], root_spec_name: str, deployment_version: str ) -> dict[str, Any]: + manifest.setdefault("spack", {}).setdefault("modules", {}).setdefault("default", {}).setdefault("tcl", {}).setdefault("projections", {}) - if custom_root_projection is not None and custom_root_projection != "": - projection_components = custom_root_projection.split("/", 1) + current_root_projection = Projections(manifest).get_projection_with_name(root_spec_name) + number_of_root_specs_in_speclist = len(Specs(manifest).get_specs_with_name(root_spec_name)) + + if current_root_projection: + # Essentially - replace the original version infix with the prX-Y style, and add back the custom suffix if there was one. + # For example: + # {name}/2025.12.000 -> {name}/prX-Y + # system-tools/{name}/2025.12.000 -> system-tools/{name}/prX-Y + # {name}/2025.12.000/{variant.x} -> {name}/prX-Y/{variant.x} + new_root_projection_pattern = re.compile( + r""" + ^(.*\{name\}/)[^/]+ # Pre-infix section - like {name} or system-tools/{name} + (/.+)?$ # Post-infix section - like /{variants.x} + """, + re.VERBOSE + ) + new_root_projection = re.sub(new_root_projection_pattern, fr"\1{deployment_version}\2", current_root_projection) - if len(projection_components) == 1: - updated_version: str = f"{{name}}/{root_spec_version}/{projection_components[0]}" - else: - updated_version: str = f"{{name}}/{root_spec_version}/{projection_components[1]}" + if number_of_root_specs_in_speclist > 1 and re.match(fr"^.+/{deployment_version}$", new_root_projection): + # If there are multiple of the same root spec, we need to demarcate them somehow if there was no custom suffix given. + # We use a short package hash as the demarcator if no custom suffix was given. + new_root_projection += "/{hash:7}" else: - updated_version: str = f"{{name}}/{root_spec_version}" - - manifest.setdefault("spack", {}).setdefault("modules", {}).setdefault("default", {}).setdefault("tcl", {}).setdefault("projections", {}) + if number_of_root_specs_in_speclist == 1: + new_root_projection = f'{{name}}/{deployment_version}' + else: + new_root_projection = f'{{name}}/{deployment_version}/{{hash:7}}' - manifest["spack"]["modules"]["default"]["tcl"]["projections"][root_spec_name] = updated_version + manifest["spack"]["modules"]["default"]["tcl"]["projections"][root_spec_name] = new_root_projection return manifest -def add_prerelease_repos_section(manifest_str: str, spack_packages_path: str) -> str: - manifest_str += ( - f" repos::\n" - f" - {spack_packages_path}\n" - f" - $spack/var/spack/repos/builtin\n" - ) +def add_prerelease_repos_section( + manifest: dict[str, Any], spack_packages_path: str, spack_packages_version_sha: str | None = None +) -> dict[str, Any]: - return manifest_str + manifest.setdefault("spack", {}).setdefault("repos", {}) + manifest["spack"]["repos"] = { + "access_spack_packages": { + "git": "https://github.com/ACCESS-NRI/access-spack-packages.git", + "destination": spack_packages_path, + } + } + + if spack_packages_version_sha: + manifest["spack"]["repos"]["access_spack_packages"]["commit"] = spack_packages_version_sha + + return manifest def parse_args(args: list[str]) -> argparse.Namespace: @@ -179,26 +179,17 @@ def parse_args(args: list[str]) -> argparse.Namespace: ) parser.add_argument( - "--custom-root-projection", + "--spack-packages-path", type=str, required=False, - help="Custom projection string to be used for the root spec in the manifest", - ) - - # This option is for the special case where the root spec defined at the repository level (a bundle with - # a version that doesn't yet exist) is not the same as the root spec defined in the manifest (which could - # be a regular package with a meaningful version). This is not recommended, but can be useful for special builds. - parser.add_argument( - "--keep-root-spec-intact", - action="store_true", - help="If set, the root spec will not be modified to remove git version information.", + help="Local path to a spack-packages repository that is added to the manifests repos section", ) parser.add_argument( - "--spack-packages-path", + "--spack-packages-version-sha", type=str, required=False, - help="Local path to a spack-packages repository that is added to the manifests repos section", + help="Git SHA of the spack-packages repository that is added to the manifests repos section", ) # Args dealing with outputs @@ -218,9 +209,8 @@ def main(): injected_manifest: str = inject_prerelease_information( args.manifest, args.version, - args.custom_root_projection, - args.keep_root_spec_intact, args.spack_packages_path, + args.spack_packages_version_sha, ) print(injected_manifest) diff --git a/scripts/spack_manifest/injection/yaml_representer.py b/scripts/spack_manifest/injection/yaml_representer.py new file mode 100644 index 00000000..f62be703 --- /dev/null +++ b/scripts/spack_manifest/injection/yaml_representer.py @@ -0,0 +1,46 @@ +from typing import Any + +# For sequences of reserved definitions, we keep it in the flow-style format (i.e., [a, b, c]) rather than block style +# as it is more compact for reserved definitions. +class YamlExplicitFlowStyleSequence(list[str]): + pass + +def yaml_explicit_flow_style_sequence_representer(dumper, data): + """ + Custom representer for YAML to ensure that some sequences are represented in flow style. + This is necessary for sequences that are used as definitions in spack manifests. + """ + return dumper.represent_sequence("tag:yaml.org,2002:seq", data, flow_style=True) + + +# PyYaml by default dumps unquoted strings if they look unambiguous, and quoted strings otherwise. +# PyYaml dumps '{name}/prX-Y' as a quoted str as it has '{' at the front and causes ambiguity +# But 'ROOT_SPEC/.dependencies/prX-Y/VERSION-{hash:7}' is dumped as an unquoted str as it is unambiguous +# So we need to wrap projections in a custom class that forces PyYaml to dump them as quoted strings. +class YamlExplicitQuotedString(str): + pass + +def yaml_explicit_quoted_string_representer(dumper, data): + """ + Custom representer for YAML to ensure that some strings are quoted explicitly. + This is necessary for strings that are used as projections in spack manifests. + """ + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="'") + +# Custom logic to enforce explicit flow-style sequences for `spack.definitions` that are reserved definitions +def enforce_explicit_flow_style_definitions(manifest: dict[str, Any]) -> dict[str, Any]: + """ + Ensure that the 'definitions' section of the manifest is represented in flow style. + This is necessary for spack manifests to ensure that definitions are correctly interpreted. + """ + if "spack" in manifest and "definitions" in manifest["spack"]: + definitions: list[dict[str, Any]] = manifest["spack"]["definitions"] + for i in range(len(definitions)): + definition = definitions[i] + if len(definition) > 0: + reserved_definition, reserved_value_list = list(definition.items())[0] + + if reserved_definition.startswith("_"): + manifest["spack"]["definitions"][i][reserved_definition] = YamlExplicitFlowStyleSequence(reserved_value_list) + + return manifest diff --git a/tests/scripts/jinja_template/inputs/deploy-outputs.Gadi b/tests/scripts/jinja_template/inputs/deploy-outputs.Gadi index 410cf29a..bd25b6ed 100644 --- a/tests/scripts/jinja_template/inputs/deploy-outputs.Gadi +++ b/tests/scripts/jinja_template/inputs/deploy-outputs.Gadi @@ -1,10 +1,19 @@ { + "model_name": "ACCESS-ISSM", + "model_deployment_repository": "https://github.com/ACCESS-NRI/ACCESS-ISSM", + "model_deployment_repository_spack_manifest_path": "spack.yaml", "spack_version": "0.22", + "spack_git_hash": "ee9a10e5cafdcd3224b436d3f477a73ef8669b4b", "spack_config_version": "2024.11.27", - "spack_packages_version": "2024.09.20", + "spack_config_git_hash": "a69211111414bd53fab72333f6340811fc52dbec", + "builtin_spack_packages_version": "2024.09.20", + "builtin_spack_packages_git_hash": "86c97e8078ab7a7dbc3497b7c937082ac5bc7c5f", + "access_spack_packages_version": "2025.10.000", + "access_spack_packages_git_hash": "86c97e8078ab7a7dbc3497b7c937082ac5bc7c5f", "ci_configuration_check_failure": false, - "release_deployment_version": "2024.11.16", - "spack_environment_name": "access-test-pr15-11", + "release_deployment_version": "2025.03.0", + "spack_environment_name": "access-issm-pr28-4", + "root_spec_pkg_hash": "qzveqwadasdmgrlp5k3zjxo7wovi5635", "deployment_result": "success", "deployment_modules_location": "/g/data/vk83/prerelease/modules", "deployment_spack_location": "/g/data/vk83/prerelease/apps/spack/0.22/spack" diff --git a/tests/scripts/jinja_template/inputs/deploy-outputs.Setonix b/tests/scripts/jinja_template/inputs/deploy-outputs.Setonix index e8a14589..acfe1e40 100644 --- a/tests/scripts/jinja_template/inputs/deploy-outputs.Setonix +++ b/tests/scripts/jinja_template/inputs/deploy-outputs.Setonix @@ -1,10 +1,19 @@ { + "model_name": "ACCESS-ISSM", + "model_deployment_repository": "https://github.com/ACCESS-NRI/ACCESS-ISSM", + "model_deployment_repository_spack_manifest_path": "spack.yaml", "spack_version": "0.22", + "spack_git_hash": "ee9a10e5cafdcd3224b436d3f477a73ef8669b4b", "spack_config_version": "2024.11.27", - "spack_packages_version": "2024.09.20", + "spack_config_git_hash": "a69211111414bd53fab72333f6340811fc52dbec", + "builtin_spack_packages_version": "2024.09.20", + "builtin_spack_packages_git_hash": "86c97e8078ab7a7dbc3497b7c937082ac5bc7c5f", + "access_spack_packages_version": "2025.10.000", + "access_spack_packages_git_hash": "86c97e8078ab7a7dbc3497b7c937082ac5bc7c5f", "ci_configuration_check_failure": false, - "release_deployment_version": "2024.11.16", - "spack_environment_name": "access-test-pr15-11", + "release_deployment_version": "2025.03.0", + "spack_environment_name": "access-issm-pr28-4", + "root_spec_pkg_hash": "qzveqwadasdmgrlp5k3zjxo7wovi5635", "deployment_result": "success", "deployment_modules_location": "/some/dir/with/modules", "deployment_spack_location": "/some/apps/spack/0.22/spack" diff --git a/tests/scripts/jinja_template/test_render_deployment_info.py b/tests/scripts/jinja_template/test_render_deployment_info.py index 8ac16160..57aef6e6 100644 --- a/tests/scripts/jinja_template/test_render_deployment_info.py +++ b/tests/scripts/jinja_template/test_render_deployment_info.py @@ -51,18 +51,20 @@ def test_build_deployments_context_from_folder(deployments_folder): assert context[0]["name"] == "Gadi" assert context[0]["result"] == "success" - assert context[0]["release_version"] == "2024.11.16" + assert context[0]["release_version"] == "2025.03.0" assert context[0]["spack_version"] == "0.22" assert context[0]["spack_config_version"] == "2024.11.27" - assert context[0]["spack_packages_version"] == "2024.09.20" + assert context[0]["builtin_spack_packages_version"] == "2024.09.20" + assert context[0]["access_spack_packages_version"] == "2025.10.000" assert context[0]["modules_location"] == "/g/data/vk83/prerelease/modules" assert context[0]["spack_location"] == "/g/data/vk83/prerelease/apps/spack/0.22/spack" assert context[1]["name"] == "Setonix" assert context[1]["result"] == "success" - assert context[1]["release_version"] == "2024.11.16" + assert context[1]["release_version"] == "2025.03.0" assert context[1]["spack_version"] == "0.22" assert context[1]["spack_config_version"] == "2024.11.27" - assert context[1]["spack_packages_version"] == "2024.09.20" + assert context[1]["builtin_spack_packages_version"] == "2024.09.20" + assert context[1]["access_spack_packages_version"] == "2025.10.000" assert context[1]["modules_location"] == "/some/dir/with/modules" assert context[1]["spack_location"] == "/some/apps/spack/0.22/spack" \ No newline at end of file diff --git a/tests/scripts/release_provenance/inputs/1-0-0/deploy-access-om3-outputs.Gadi/deploy-access-om3-outputs.Gadi b/tests/scripts/release_provenance/inputs/deploy-access-om3-outputs.Gadi/deploy-access-om3-outputs.Gadi similarity index 76% rename from tests/scripts/release_provenance/inputs/1-0-0/deploy-access-om3-outputs.Gadi/deploy-access-om3-outputs.Gadi rename to tests/scripts/release_provenance/inputs/deploy-access-om3-outputs.Gadi/deploy-access-om3-outputs.Gadi index 273307b0..adf87ac4 100644 --- a/tests/scripts/release_provenance/inputs/1-0-0/deploy-access-om3-outputs.Gadi/deploy-access-om3-outputs.Gadi +++ b/tests/scripts/release_provenance/inputs/deploy-access-om3-outputs.Gadi/deploy-access-om3-outputs.Gadi @@ -5,8 +5,10 @@ "spack_git_hash": "qwerty", "spack_config_version": "2025.02.2", "spack_config_git_hash": "asdfg", - "spack_packages_version": "2025.03.002", - "spack_packages_git_hash": "zxcvb", + "builtin_spack_packages_version": "2025.03.002", + "builtin_spack_packages_git_hash": "zxcvb", + "access_spack_packages_version": "2025.10.000", + "access_spack_packages_git_hash": "asdfg", "ci_configuration_check_failure": false, "release_deployment_version": "2025.01.2", "spack_environment_name": "access-esm1p6-dev_2025_03_4", diff --git a/tests/scripts/release_provenance/inputs/1-0-0/deploy-access-test-metadata.Gadi/Gadi.build-db-pkgs.json b/tests/scripts/release_provenance/inputs/deploy-access-test-metadata.Gadi/Gadi.build-db-pkgs.json similarity index 100% rename from tests/scripts/release_provenance/inputs/1-0-0/deploy-access-test-metadata.Gadi/Gadi.build-db-pkgs.json rename to tests/scripts/release_provenance/inputs/deploy-access-test-metadata.Gadi/Gadi.build-db-pkgs.json diff --git a/tests/scripts/release_provenance/test_tracking_services_data.py b/tests/scripts/release_provenance/test_tracking_services_data.py index 7a81f0bc..afd0c32d 100644 --- a/tests/scripts/release_provenance/test_tracking_services_data.py +++ b/tests/scripts/release_provenance/test_tracking_services_data.py @@ -118,7 +118,7 @@ def test__format_tracking_services_header__valid(self): class TestFormatTelemetryOfModel(): @patch("scripts.release_provenance.tracking_services_data._get_release_data_of_model_or_raise") - def test__format_telemetry_of_model_1_0_0__valid(self, release_data_mock): + def test__format_telemetry_of_model__valid(self, release_data_mock): expected_telemetry = { "model.name": "ACCESS-OM3", "model.deployment_repository_url": "https://github.com/ACCESS-NRI/ACCESS-OM3", @@ -135,25 +135,25 @@ def test__format_telemetry_of_model_1_0_0__valid(self, release_data_mock): "html_url": "https://github.com/ACCESS-NRI/ACCESS-OM3/releases/tag/2025.01.2", } - actual_telemetry = _format_telemetry_of_model("ACCESS-NRI/ACCESS-OM3", Path("tests/scripts/release_provenance/inputs/1-0-0/deploy-access-om3-outputs.Gadi")) + actual_telemetry = _format_telemetry_of_model("ACCESS-NRI/ACCESS-OM3", Path("tests/scripts/release_provenance/inputs/deploy-access-om3-outputs.Gadi")) assert actual_telemetry == expected_telemetry, f"Expected {expected_telemetry}, but got {actual_telemetry}" @patch("scripts.release_provenance.tracking_services_data.requests.get", autospec=True) - def test__format_telemetry_of_model_1_0_0__invalid_repo(self, requests_mock): + def test__format_telemetry_of_model__invalid_repo(self, requests_mock): # Mock the response of the requests.get call to simulate an invalid repository requests_mock.return_value = Mock(status_code=404) with pytest.raises(ValueError): - _format_telemetry_of_model("invalid-repo", Path("tests/scripts/release_provenance/inputs/1-0-0/deploy-access-om3-outputs.Gadi")) + _format_telemetry_of_model("invalid-repo", Path("tests/scripts/release_provenance/inputs/deploy-access-om3-outputs.Gadi")) - def test__format_telemetry_of_model_1_0_0__invalid_file(self): + def test__format_telemetry_of_model_invalid_file(self): with pytest.raises(FileNotFoundError): _format_telemetry_of_model("ACCESS-NRI/ACCESS-OM3", Path("invalid-file-path")) class TestFormatTelemetryOfDeploymentTarget(): - def test__format_telemetry_of_deployment_target_1_0_0__valid_with_no_components(self): + def test__format_telemetry_of_deployment_target___valid_with_no_components(self): # Removing the components from the expected telemetry as that is tested in TestFormatTelemetryOfModelComponents expected_telemetry_without_components = { "deployment_target.name": "Gadi", @@ -161,8 +161,10 @@ def test__format_telemetry_of_deployment_target_1_0_0__valid_with_no_components( "deployment_target.spack_git_hash": "qwerty", "deployment_target.spack_config_version": "2025.02.2", "deployment_target.spack_config_git_hash": "asdfg", - "deployment_target.spack_packages_version": "2025.03.002", - "deployment_target.spack_packages_git_hash": "zxcvb", + "deployment_target.builtin_spack_packages_version": "2025.03.002", + "deployment_target.builtin_spack_packages_git_hash": "zxcvb", + "deployment_target.access_spack_packages_version": "2025.10.000", + "deployment_target.access_spack_packages_git_hash": "asdfg", "deployment_target.module_use_location": "/g/data/vk83/modules", "spack_model.name": "access-om3", @@ -170,13 +172,13 @@ def test__format_telemetry_of_deployment_target_1_0_0__valid_with_no_components( "spack_model.module_load_command": "access-om3/2025.01.2", } - actual_telemetry = _format_telemetry_of_deployment_target("access-om3", "Gadi", Path("tests/scripts/release_provenance/inputs/1-0-0/deploy-access-om3-outputs.Gadi"), Path("tests/scripts/release_provenance/inputs/1-0-0/deploy-access-test-metadata.Gadi")) + actual_telemetry = _format_telemetry_of_deployment_target("access-om3", "Gadi", Path("tests/scripts/release_provenance/inputs/deploy-access-om3-outputs.Gadi"), Path("tests/scripts/release_provenance/inputs/deploy-access-test-metadata.Gadi")) actual_telemetry_without_components = {k: v for k, v in actual_telemetry.items() if k not in ["spack_model_components"]} assert actual_telemetry_without_components == expected_telemetry_without_components, f"Without components, expected {expected_telemetry_without_components}, but got {actual_telemetry_without_components}" class TestFormatTelemetryOfModelComponents(): - def test__format_telemetry_of_model_components_1_0_0__valid(self): + def test__format_telemetry_of_model_components__valid(self): expected_telemetry = [ { "name": "access-test-component", @@ -187,7 +189,7 @@ def test__format_telemetry_of_model_components_1_0_0__valid(self): }, ] - actual_telemetry = _format_telemetry_of_model_components("Gadi", Path("tests/scripts/release_provenance/inputs/1-0-0/deploy-access-test-metadata.Gadi")) + actual_telemetry = _format_telemetry_of_model_components("Gadi", Path("tests/scripts/release_provenance/inputs/deploy-access-test-metadata.Gadi")) assert actual_telemetry == expected_telemetry, f"Expected {expected_telemetry}, but got {actual_telemetry}" diff --git a/tests/scripts/spack_manifest/injection/inputs/prerelease.spack.yaml b/tests/scripts/spack_manifest/injection/inputs/prerelease.spack.yaml index dc44818d..71638bfa 100644 --- a/tests/scripts/spack_manifest/injection/inputs/prerelease.spack.yaml +++ b/tests/scripts/spack_manifest/injection/inputs/prerelease.spack.yaml @@ -4,37 +4,40 @@ # configuration settings. spack: # add package specs to the `specs` list + definitions: + - _name: [access-om2] + - _version: [2024.03.0] specs: - - access-om2@git.2024.03.0=latest + - access-om2 packages: cice5: require: - - '@git.2023.10.19=access-om2' + - '@git.2023.10.19=access-om2' mom5: require: - - '@git.2023.11.09=access-om2' + - '@git.2023.11.09=access-om2' libaccessom2: require: - - '@git.2023.10.26=access-om2' + - '@git.2023.10.26=access-om2' oasis3-mct: require: - - '@git.2023.11.09=access-om2' + - '@git.2023.11.09=access-om2' netcdf-c: require: - - '@4.7.4' + - '@4.7.4' netcdf-fortran: require: - - '@4.5.2' + - '@4.5.2' parallelio: require: - - '@2.5.2' + - '@2.5.2' openmpi: require: - - '@4.0.2' + - '@4.0.2' all: require: - - '%intel@19.0.5.281' - - 'target=x86_64' + - '%intel@19.0.5.281' + - 'target=x86_64' view: true concretizer: unify: true @@ -42,11 +45,11 @@ spack: default: tcl: include: - - access-om2 - - mom5 - - cice5 - - libaccessom2 - - oasis3-mct + - access-om2 + - mom5 + - cice5 + - libaccessom2 + - oasis3-mct projections: access-om2: '{name}/2024.03.0' mom5: '{name}/2023.11.09-{hash:7}' diff --git a/tests/scripts/spack_manifest/injection/inputs/spack.yaml b/tests/scripts/spack_manifest/injection/inputs/spack.yaml index c9626f08..d4f1825d 100644 --- a/tests/scripts/spack_manifest/injection/inputs/spack.yaml +++ b/tests/scripts/spack_manifest/injection/inputs/spack.yaml @@ -4,8 +4,11 @@ # configuration settings. spack: # add package specs to the `specs` list + definitions: + - _name: ["access-om2"] + - _version: ["2024.03.0"] specs: - - access-om2@git.2024.03.0=latest + - access-om2 packages: cice5: require: diff --git a/tests/scripts/spack_manifest/injection/outputs/expected.prerelease.spack.yaml b/tests/scripts/spack_manifest/injection/outputs/expected.prerelease.spack.yaml index fedc2bb5..e135f0d2 100644 --- a/tests/scripts/spack_manifest/injection/outputs/expected.prerelease.spack.yaml +++ b/tests/scripts/spack_manifest/injection/outputs/expected.prerelease.spack.yaml @@ -1,4 +1,7 @@ spack: + definitions: + - _name: [access-om2] + - _version: [2024.03.0] specs: - access-om2 packages: @@ -43,8 +46,10 @@ spack: - libaccessom2 - oasis3-mct projections: - access-om2: '{name}/pr12-12/2024.03.0' + access-om2: '{name}/pr12-12' mom5: 'access-om2/dependencies/pr12-12/{name}/2023.11.09-{hash:7}' - repos:: - - /some/spack-packages - - $spack/var/spack/repos/builtin + repos: + access_spack_packages: + git: https://github.com/ACCESS-NRI/access-spack-packages.git + destination: /some/spack-packages + commit: e8713551c6eee57caf9603543e6dd6daf3c93922 diff --git a/tests/scripts/spack_manifest/injection/test_modules.py b/tests/scripts/spack_manifest/injection/test_modules.py index b7cd8714..fda1c90e 100644 --- a/tests/scripts/spack_manifest/injection/test_modules.py +++ b/tests/scripts/spack_manifest/injection/test_modules.py @@ -37,13 +37,13 @@ def test_parse_args__valid_no_optionals_multiple_packages(self): "--manifest", "tests/scripts/spack_manifest/injection/inputs/spack.yaml", "--packages", - "mom5 cice5 libaccessom2" + "mom5,cice5,libaccessom2" ] parsed_args = parse_args(args) assert parsed_args.manifest == "tests/scripts/spack_manifest/injection/inputs/spack.yaml" - assert parsed_args.packages == "mom5 cice5 libaccessom2" + assert parsed_args.packages == "mom5,cice5,libaccessom2" assert parsed_args.output is None, "Output should be None when not specified." @@ -54,31 +54,20 @@ def test_parse_args__valid_with_optionals_multiple_packages(self): "--output", "output.yaml", "--packages", - "mom5 cice5 libaccessom2" + "mom5,cice5,libaccessom2" ] parsed_args = parse_args(args) assert parsed_args.manifest == "tests/scripts/spack_manifest/injection/inputs/spack.yaml" assert parsed_args.output == "output.yaml" - assert parsed_args.packages == "mom5 cice5 libaccessom2" - - def test_parse_args__invalid_comma_separated_packages(self): - args = [ - "--manifest", - "tests/scripts/spack_manifest/injection/inputs/spack.yaml", - "--packages", - "mom5,cice5,libaccessom2" - ] - - with pytest.raises(ValueError): - parse_args(args) + assert parsed_args.packages == "mom5,cice5,libaccessom2" class TestGenerateProjectionForRootSpecOrRaise: def test_generate_projection_for_root_spec_or_raise__valid_single_target(self): - manifest = {"spack": {"specs": ["access-om2@git.2025.05.000 +debug ~mpi"]}} + manifest = {"spack": {"definitions": [{"_name": ["access-om2"]}, {"_version": ["2025.05.000"]}],"specs": ["access-om2 +debug ~mpi"]}} projection = "access-om2" expected_version = {"access-om2": "{name}/2025.05.000"} @@ -92,6 +81,8 @@ def test_generate_projection_for_root_spec_or_raise__valid_multi_target(self): manifest = { "spack": { "definitions": [ + {"_name": ["access-om2"]}, + {"_version": ["2025.05.000"]}, {"ROOT_PACKAGE": ["access-om2@git.2025.05.000 +debug ~mpi"]} ] } @@ -108,7 +99,7 @@ def test_generate_projection_for_root_spec_or_raise__valid_multi_target(self): def test_generate_projection_for_root_spec_or_raise__single_target_no_version_defined( self, ): - manifest = {"spack": {"specs": ["access-om2"]}} + manifest = {"spack": {"definitions": [{"_name": ["access-om2"]}],"specs": ["access-om2"]}} projection = "access-om2" @@ -128,7 +119,7 @@ def test_generate_projection_for_root_spec_or_raise__multi_target_no_version_def def test_generate_projection_for_root_spec_or_raise__single_target_wrong_projection( self, ): - manifest = {"spack": {"specs": ["access-om2@git.2025.05.000 +debug ~mpi"]}} + manifest = {"spack": {"definitions": [{"_name": ["access-om2"]}, {"_version": ["2025.05.000"]}],"specs": ["access-om2 +debug ~mpi"]}} projection = "wrong-projection" @@ -141,7 +132,9 @@ def test_generate_projection_for_root_spec_or_raise__multi_target_wrong_projecti manifest = { "spack": { "definitions": [ - {"ROOT_PACKAGE": ["access-om2@git.2025.05.000 +debug ~mpi"]} + {"_name": ["access-om2"]}, + {"_version": ["2025.05.000"]}, + {"ROOT_PACKAGE": ["access-om2@git.2025.05.000 +debug ~mpi"]}, ] } } @@ -206,7 +199,11 @@ def test_inject_projections__valid(self): expected_output = { "spack": { - "specs": ["access-om2@git.2024.03.0=latest"], + "definitions": [ + {"_name": ["access-om2"]}, + {"_version": ["2024.03.0"]}, + ], + "specs": ["access-om2"], "packages": { "cice5": {"require": ["@git.2023.10.19=access-om2"]}, "mom5": {"require": ["@git.2023.11.09=access-om2"]}, diff --git a/tests/scripts/spack_manifest/injection/test_prerelease.py b/tests/scripts/spack_manifest/injection/test_prerelease.py index 750a25e4..1be0a2e0 100644 --- a/tests/scripts/spack_manifest/injection/test_prerelease.py +++ b/tests/scripts/spack_manifest/injection/test_prerelease.py @@ -3,22 +3,24 @@ from scripts.spack_manifest.injection.prerelease import ( inject_prerelease_information, add_prerelease_repos_section, - remove_potential_root_spec_git_version, update_root_spec_projection_version, add_namespace_to_other_projection_versions, parse_args ) class TestUpdateRootSpecProjectionVersion: - def test_update_root_spec_projection_version__valid_existing(self): + def test_update_root_spec_projection_version__valid_existing_single_spec(self): manifest = { "spack": { + "specs": [ + "access-om2", + ], "modules": { "default": { "tcl": { "projections": { - "access-om2": "{name}/1.0.0", - "dependency": "{name}/2.0.0" + "access-om2": "{name}/2025.12.000/1.0.0", + "dependency": "{name}/2025.12.000/2.0.0" } } } @@ -27,19 +29,25 @@ def test_update_root_spec_projection_version__valid_existing(self): } root_spec_name = "access-om2" root_spec_version = "pr12-2" - custom_root_spec_projection = "1.0.0" - updated_manifest = update_root_spec_projection_version(manifest, root_spec_name, root_spec_version, custom_root_spec_projection) + updated_manifest = update_root_spec_projection_version(manifest, root_spec_name, root_spec_version) - assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"][root_spec_name] == f"{{name}}/{root_spec_version}/{custom_root_spec_projection}" + assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"][root_spec_name] == f"{{name}}/{root_spec_version}/1.0.0" - def test_update_root_spec_projection_version__valid_new(self): + def test_update_root_spec_projection_version__valid_existing_multi_spec(self): manifest = { "spack": { + "specs": [ + "access-om2 +var", + "access-om2 ~var" + ], "modules": { "default": { "tcl": { - "projections": {} + "projections": { + "access-om2": "{name}/2025.12.000/1.0.0", + "dependency": "{name}/2025.12.000/2.0.0" + } } } } @@ -50,14 +58,20 @@ def test_update_root_spec_projection_version__valid_new(self): updated_manifest = update_root_spec_projection_version(manifest, root_spec_name, root_spec_version) - assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"][root_spec_name] == f"{{name}}/{root_spec_version}" + assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"][root_spec_name] == f"{{name}}/{root_spec_version}/1.0.0" - def test_update_root_spec_projection_version__no_projections(self): + + def test_update_root_spec_projection_version__valid_new_single_spec(self): manifest = { "spack": { + "specs": [ + "access-om2" + ], "modules": { "default": { - "tcl": {} + "tcl": { + "projections": {} + } } } } @@ -69,47 +83,72 @@ def test_update_root_spec_projection_version__no_projections(self): assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"][root_spec_name] == f"{{name}}/{root_spec_version}" -class TestRemovePotentialRootSpecGitVersion: - def test_remove_potential_root_spec_git_version__valid_no_constraints(self): + def test_update_root_spec_projection_version__valid_new_multi_spec(self): manifest = { "spack": { "specs": [ - "access-om2@git.1.0.0" - ] + "access-om2 +var", + "access-om2 ~var" + ], + "modules": { + "default": { + "tcl": { + "projections": {} + } + } + } } } + root_spec_name = "access-om2" + root_spec_version = "pr12-2" - updated_manifest = remove_potential_root_spec_git_version(manifest) + updated_manifest = update_root_spec_projection_version(manifest, root_spec_name, root_spec_version) - assert updated_manifest["spack"]["specs"][0] == "access-om2" + assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"][root_spec_name] == f"{{name}}/{root_spec_version}/{{hash:7}}" - def test_remove_potential_root_spec_git_version__valid_with_constraints_no_space(self): + def test_update_root_spec_projection_version__no_projections_single_spec(self): manifest = { "spack": { "specs": [ - r"access-om2@git.1.0.0%compiler@2.0.0" - ] + "access-om2" + ], + "modules": { + "default": { + "tcl": {} + } + } } } + root_spec_name = "access-om2" + root_spec_version = "pr12-2" - updated_manifest = remove_potential_root_spec_git_version(manifest) + updated_manifest = update_root_spec_projection_version(manifest, root_spec_name, root_spec_version) - assert updated_manifest["spack"]["specs"][0] == f"access-om2 %compiler@2.0.0" + assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"][root_spec_name] == f"{{name}}/{root_spec_version}" - def test_remove_potential_root_spec_git_version__valid_with_constraints_with_space(self): + def test_update_root_spec_projection_version__no_projections_multi_spec(self): manifest = { "spack": { "specs": [ - r"access-om2@git.1.0.0 %compiler@2.0.0 +debug" - ] + "access-om2 +var", + "access-om2 ~var" + ], + "modules": { + "default": { + "tcl": {} + } + } } } + root_spec_name = "access-om2" + root_spec_version = "pr12-2" - updated_manifest = remove_potential_root_spec_git_version(manifest) + updated_manifest = update_root_spec_projection_version(manifest, root_spec_name, root_spec_version) - assert updated_manifest["spack"]["specs"][0] == f"access-om2 %compiler@2.0.0 +debug" + assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"][root_spec_name] == f"{{name}}/{root_spec_version}/{{hash:7}}" - def test_remove_potential_root_spec_git_version__invalid_no_git_version(self): +class TestAddPrereleaseReposSection: + def test_add_prerelease_repos_section__valid(self): manifest = { "spack": { "specs": [ @@ -117,37 +156,27 @@ def test_remove_potential_root_spec_git_version__invalid_no_git_version(self): ] } } - - updated_manifest = dict(remove_potential_root_spec_git_version(manifest)) - - assert manifest == updated_manifest, "Manifest should remain unchanged if no git version is present" - -class TestAddPrereleaseReposSection: - def test_add_prerelease_repos_section__valid(self): - manifest_str = "spack:\n specs:\n - access-om2@git.1.0.0\n" spack_packages_path = "/path/to/spack/packages" - updated_manifest_str = add_prerelease_repos_section(manifest_str, spack_packages_path) - - last_three_lines = "\n".join(updated_manifest_str.splitlines()[-3:]) - - expected_last_three_lines = ( - f" repos::\n" - f" - {spack_packages_path}\n" - f" - $spack/var/spack/repos/builtin\n" - ) + updated_manifest = add_prerelease_repos_section(manifest, spack_packages_path) - assert last_three_lines.strip() == expected_last_three_lines.strip() + expected_repos_section = { + "access_spack_packages": { + "git": "https://github.com/ACCESS-NRI/access-spack-packages.git", + "destination": spack_packages_path, + } + } + assert updated_manifest["spack"]["repos"] == expected_repos_section class TestInjectPrereleaseInformation: def test_inject_prerelease_information__valid_custom_projection(self): manifest_path = "tests/scripts/spack_manifest/injection/inputs/prerelease.spack.yaml" root_spec_version = "pr12-12" - custom_root_projection = "2024.03.0" spack_packages_path = "/some/spack-packages" + spack_packages_version_sha = "e8713551c6eee57caf9603543e6dd6daf3c93922" updated_manifest_str: str = inject_prerelease_information( - manifest_path, root_spec_version, custom_root_projection, False, spack_packages_path + manifest_path, root_spec_version, spack_packages_path, spack_packages_version_sha ) expected_manifest_path = "tests/scripts/spack_manifest/injection/outputs/expected.prerelease.spack.yaml" diff --git a/tests/scripts/spack_manifest/test_getter.py b/tests/scripts/spack_manifest/test_getter.py index 537c8ae4..80828e92 100644 --- a/tests/scripts/spack_manifest/test_getter.py +++ b/tests/scripts/spack_manifest/test_getter.py @@ -2,10 +2,12 @@ import yaml from scripts.spack_manifest.getter import ( + ReservedDefinitions, RootSpec, Packages, Includes, Projections, + Specs, NoSectionError, NoSectionComponentError, ) @@ -36,6 +38,55 @@ def manifest_from_file(): } } +class TestReservedDefinitionsGetter: + @pytest.fixture + def manifest_with_reserved_definitions(self): + return { + "spack": { + "definitions": [ + {"_name": ["access-om2"]}, + {"_version": ["2025.11.000"]}, + {"OTHER_DEFINITION": ["some-value"]}, + ] + } + } + def test___init___valid(self, manifest_with_reserved_definitions): + + reserved_definitions_getter = ReservedDefinitions(manifest_with_reserved_definitions) + + expected = { + "name": "access-om2", + "version": "2025.11.000", + } + + assert ( + reserved_definitions_getter.reserved_definitions == expected + ), "Reserved definitions should be correctly initialized from manifest." + + def test___init___invalid_no_definitions_section(self): + manifest = {"spack": {}} + + with pytest.raises(NoSectionError): + ReservedDefinitions(manifest) + + def test_get__valid(self, manifest_with_reserved_definitions): + + reserved_definitions_getter = ReservedDefinitions(manifest_with_reserved_definitions) + definition_value = reserved_definitions_getter.get("name") + + expected = "access-om2" + + assert ( + definition_value == expected + ), "Reserved definitions should be correctly retrieved." + + def test_get__invalid_no_definition(self, manifest_with_reserved_definitions): + + reserved_definitions_getter = ReservedDefinitions(manifest_with_reserved_definitions) + + with pytest.raises(NoSectionComponentError): + reserved_definitions_getter.get("nonexistent_definition") + class TestRootSpecGetter: def test___init___valid_multi_target_spec(self): @@ -448,3 +499,41 @@ def test__get_defined_includes__no_modules_section(self): assert ( result == expected ), "Manifest without a modules section should return an empty set." + + +class TestSpecsGetter: + @pytest.mark.parametrize( + "specs", + [ + ["access-om2"], # Single root spec + ["access-om2 +var", "access-om2 ~var"] # Multiple root specs + ] + ) + def test_get_specs__valid(self, specs): + manifest = {"spack": {"specs": specs}} + + assert Specs(manifest).get_specs() == specs, "Should return all specs" + + @pytest.mark.parametrize( + "specs,expected", + [ + (["access-om2", "access-om3"], ["access-om2"]), # Single root spec + (["access-om2 +var", "access-om2 ~var", "access-om3"], ["access-om2 +var", "access-om2 ~var"]) # Multiple root specs + ] + ) + def test_get_specs_with_name__exist(self, specs, expected): + manifest = {"spack": {"specs": specs}} + + assert Specs(manifest).get_specs_with_name("access-om2") == expected, "Should return specs with the given name" + + @pytest.mark.parametrize( + "specs", + [ + ["access-om3"], + ["access-om3 +var", "access-om3 ~var"] + ] + ) + def test_get_specs_with_name__no_exist(self, specs): + manifest = {"spack": {"specs": specs}} + + assert Specs(manifest).get_specs_with_name("access-om2") == [], "Should return no specs" \ No newline at end of file