diff --git a/.github/actions/get-pr-deployments/README.md b/.github/actions/get-pr-deployments/README.md
new file mode 100644
index 00000000..1711f243
--- /dev/null
+++ b/.github/actions/get-pr-deployments/README.md
@@ -0,0 +1,33 @@
+# Get PR Deployments
+
+Action that returns the number of deployments on a given PR branch.
+
+## Inputs
+
+| Name | Description | Required | Default | Example |
+| ---- | ----------- | -------- | ------- | ------- |
+| `pr` | The pull request number to check for deployments of all kinds | `true` | N/A | `21` |
+| `repository` | The repository to check for deployments | `false` | Value of `github.repository` | `"ACCESS-NRI/ACCESS-OM2"` |
+| `token` | The GitHub token to use for API requests | `false` | Value of `github.token` | `"ghp_XXXX"` |
+
+## Outputs
+
+| Name | Description | Example |
+| ---- | ----------- | ------- |
+| `deployments` | The total number of deployments for the given PR (from commits and `!redeploy`s) | `24` |
+
+## Example
+
+```yaml
+# ...
+jobs:
+ deployments:
+ runs-on: ubuntu-latest
+ steps:
+ - id: pr-deploys
+ uses: access-nri/build-cd/.github/actions/get-pr-deployments@v6
+ with:
+ pr: 12
+
+ - run: echo "There have been ${{ steps.pr-deploys.outputs.deployments }} total deployments in this PR, including both regular commit deployments and redeployments"
+```
diff --git a/.github/actions/get-pr-deployments/action.yml b/.github/actions/get-pr-deployments/action.yml
new file mode 100644
index 00000000..fc778bbd
--- /dev/null
+++ b/.github/actions/get-pr-deployments/action.yml
@@ -0,0 +1,71 @@
+name: Get PR Deployments
+description: Action that returns how many deployments were made in a PR (both from commits and !redeploys)
+author: Tommy Gatti
+inputs:
+ pr:
+ required: true
+ description: The pull request number to check for deployments of all kinds
+ repository:
+ required: false
+ default: ${{ github.repository }}
+ description: The repository to check for deployments
+ token:
+ required: false
+ default: ${{ github.token }}
+ description: The GitHub token to use for API requests
+outputs:
+ deployments:
+ description: The total number of deployments for the given PR (from commits and !redeploys)
+ value: ${{ steps.total.outputs.number }}
+runs:
+ using: composite
+ # Essentially, count all the deployment entries that match the given branch, as well as
+ # all the `!redeploy` comments, to get the next deployment number.
+ # See https://docs.github.com/en/rest/deployments/deployments?apiVersion=2022-11-28#list-deployments
+ steps:
+ - name: Get PR HEAD
+ id: pr
+ shell: bash
+ env:
+ GH_TOKEN: ${{ inputs.token }}
+ run: |
+ head=$(gh pr view ${{ inputs.pr }} --repo ${{ inputs.repository }} --json headRefName --jq .headRefName)
+ echo "HEAD of PR #${{ inputs.pr }} in ${{ inputs.repository }} is $head"
+ echo "head=$head" >> $GITHUB_OUTPUT
+
+ - name: Get commit deployments for PR
+ id: commit
+ shell: bash
+ env:
+ GH_TOKEN: ${{ inputs.token }}
+ # We --slurp the results because --paginate introduces potentially multiple array results
+ run: |
+ deployments=$(gh api \
+ -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
+ --paginate --slurp \
+ /repos/${{ inputs.repository }}/deployments \
+ | jq '[select(.[][].ref == "${{ steps.pr.outputs.head }}")] | length'
+ )
+ echo "Found $deployments deployments for PR #${{ inputs.pr }} in ${{ inputs.repository }}"
+ echo "deployments=$deployments" >> $GITHUB_OUTPUT
+
+ - name: Get !redeploys from comments
+ id: comment
+ shell: bash
+ env:
+ GH_TOKEN: ${{ inputs.token }}
+ run: |
+ redeployments=$(gh pr view ${{ inputs.pr }} --repo ${{ github.repository }} \
+ --json comments \
+ --jq '[.comments[] | select(.body | startswith("!redeploy"))] | length'
+ )
+ echo "Found $redeployments redeploy comments for PR #${{ inputs.pr }} in ${{ inputs.repository }}"
+ echo "redeployments=$redeployments" >> $GITHUB_OUTPUT
+
+ - name: Return total deployments
+ id: total
+ shell: bash
+ run: |
+ total=$(( ${{ steps.commit.outputs.deployments }} + ${{ steps.comment.outputs.redeployments }} ))
+ echo "Found $total deployments across commits (${{ steps.commit.outputs.deployments }}) and redeploys (${{ steps.comment.outputs.redeployments }}) for PR #${{ inputs.pr }} in ${{ inputs.repository }}"
+ echo "number=$total" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
index efb379ef..28dcc7b5 100644
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -91,7 +91,7 @@ jobs:
- name: Generate Deployment Target Matrix
id: target
- uses: access-nri/build-cd/.github/actions/get-target-matrix@v6
+ uses: access-nri/build-cd/.github/actions/get-target-matrix@v7
with:
targets: ${{ vars.RELEASE_DEPLOYMENT_TARGETS }}
@@ -108,7 +108,7 @@ jobs:
with:
repository: access-nri/build-cd
- - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v6
+ - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v7
with:
settings-path: ./config/settings.json
target: ${{ matrix.target }}
@@ -126,7 +126,7 @@ jobs:
- name: Get root spec ref
id: tag
- uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v6
+ uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7
with:
spack-manifest-path: ${{ inputs.spack-manifest-path }}
@@ -180,7 +180,7 @@ jobs:
strategy:
matrix:
target: ${{ fromJson(needs.defaults.outputs.targets) }}
- uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v6
+ uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v7
with:
deployment-target: ${{ matrix.target }}
deployment-ref: ${{ github.ref_name }}
diff --git a/.github/workflows/ci-closed.yml b/.github/workflows/ci-closed.yml
index afe51791..d96aedb3 100644
--- a/.github/workflows/ci-closed.yml
+++ b/.github/workflows/ci-closed.yml
@@ -38,7 +38,7 @@ jobs:
- name: Generate Deployment Target Matrix
id: target
- uses: access-nri/build-cd/.github/actions/get-target-matrix@v6
+ uses: access-nri/build-cd/.github/actions/get-target-matrix@v7
with:
targets: ${{ vars.PRERELEASE_DEPLOYMENT_TARGETS }}
@@ -50,7 +50,7 @@ jobs:
matrix:
target: ${{ fromJson(needs.setup.outputs.targets) }}
fail-fast: false
- uses: access-nri/build-cd/.github/workflows/undeploy-1-start.yml@v6
+ uses: access-nri/build-cd/.github/workflows/undeploy-1-start.yml@v7
with:
version-pattern: ${{ inputs.root-sbd }}-pr${{ github.event.pull_request.number }}-*
target: ${{ matrix.target }}
diff --git a/.github/workflows/ci-command-configs.yml b/.github/workflows/ci-command-configs.yml
new file mode 100644
index 00000000..3f1d97cc
--- /dev/null
+++ b/.github/workflows/ci-command-configs.yml
@@ -0,0 +1,359 @@
+name: Configs
+on:
+ workflow_call:
+ inputs:
+ model:
+ type: string
+ required: true
+ description: The model that is being tested and deployed
+ root-sbd:
+ type: string
+ required: false
+ # The equivalent default is set in the defaults job below.
+ # default: ${{ inputs.model }}
+ description: |
+ The name of the root Spack Bundle Definition, if it is different from the model name.
+ This is often a package named similarly in ACCESS-NRI/spack-packages.
+ auto-configs-pr-schema-version:
+ type: string
+ required: true
+ description: The version of the auto-configs-pr schema to use for validation, from the ACCESS-NRI/schema repository.
+ secrets:
+ configs-repo-token:
+ required: true
+ description: |
+ The GitHub token to use for pushing and pull request creation in the *-configs repositories.
+ Requires pull-requests:write and contents:write permissions.
+ commit-gpg-private-key:
+ required: true
+ description: The private GPG key used to sign commits.
+ commit-gpg-passphrase:
+ required: true
+ description: The passphrase for the private GPG key used to sign commits.
+env:
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ GH_TOKEN: ${{ github.token }}
+ PYTHON_VERSION: '3.11'
+jobs:
+ setup:
+ name: Setup
+ # Job is used to check permissions, parse the given !update-configs command,
+ # and read configuration information from the caller repository.
+ runs-on: ubuntu-latest
+ outputs:
+ # Value for the defaulted root-sbd if not provided
+ root-sbd: ${{ steps.defaults.outputs.root-sbd }}
+ # Caller PR branch ref
+ pr-branch: ${{ steps.pr.outputs.ref }}
+ # The profile to use from the MDRs config/auto-configs-pr.json
+ profile: ${{ steps.parse.outputs.profile }}
+ # The configs to update from the MDRs config/auto-configs-pr.json, in a GitHub Actions matrix-parseable JSON array
+ configs: ${{ steps.read-config.outputs.configs }}
+ # The configs to update, in a space-separated string
+ configs-formatted: ${{ steps.read-config.outputs.configs-formatted }}
+ # The repository containing the configs to update
+ configs-repo: ${{ steps.read-config.outputs.configs-repo }}
+ steps:
+ - name: Check commenter permissions
+ id: commenter
+ uses: access-nri/actions/.github/actions/commenter-permission-check@main
+ with:
+ # This means that commenters who use `!update-configs` must have at least `write` perms
+ # in the repository.
+ minimum-permission: write
+
+ - name: React to Comment
+ uses: access-nri/actions/.github/actions/react-to-comment@main
+ with:
+ token: ${{ github.token }}
+ reaction: ${{ steps.commenter.outputs.has-permission == 'true' && 'rocket' || '-1' }}
+
+ - name: Exit if no permission to run
+ if: steps.commenter.outputs.has-permission != 'true'
+ run: |
+ echo "::error::Commenter does not have permission to run this command, requires at least 'write' permission"
+ exit 1
+
+ - name: Parse command
+ id: parse
+ env:
+ USAGE: '!update-configs [profile=PROFILE]'
+ # We don't like ! event expansion here
+ shell: bash +H {0}
+ run: |
+ # Defaults
+ profile="default"
+
+ # Parse comment into command args
+ comment='${{ github.event.comment.body }}'
+ read -r command <<< "$comment"
+ args=${command#!update-configs}
+
+ # Processing args
+ # FIXME: Wouldn't handle positional args, if they were part of the command syntax
+ # If there isn't anything after !update-configs, args will be empty (or erroneously spaces, which we strip)
+ if [ -n "${args// }" ]; then
+ for arg in $args; do
+ case $arg in
+ profile=*) profile="${arg#profile=}" ;;
+ *) echo "::error::Unknown argument '$arg'. Usage: $USAGE"; exit 1;;
+ esac
+ done
+ fi
+
+ # Output processed args
+ echo "profile=$profile" >> $GITHUB_OUTPUT
+
+ - name: Set defaults
+ id: defaults
+ run: |
+ if [[ "${{ inputs.root-sbd }}" == "" ]]; then
+ echo "root-sbd=${{ inputs.model }}" >> $GITHUB_OUTPUT
+ else
+ echo "root-sbd=${{ inputs.root-sbd }}" >> $GITHUB_OUTPUT
+ fi
+
+ - name: Get caller repository ref
+ id: pr
+ # We need to find the PR ref of the caller repository explicitly, as this workflow trigger (issue_comment) is in the context of the default branch
+ run: echo "ref=$(gh pr view ${{ github.event.issue.number }} --repo ${{ github.repository }} --json headRefName --jq .headRefName)" >> $GITHUB_OUTPUT
+
+ - name: Checkout caller repository
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ steps.pr.outputs.ref }}
+ path: caller
+
+ - name: Validate auto configs PR configuration
+ uses: access-nri/schema/.github/actions/validate-with-schema@main
+ with:
+ schema-version: ${{ inputs.auto-configs-pr-schema-version }}
+ schema-location: au.org.access-nri/model/deployment/config/auto-configs-pr
+ data-location: caller/config/auto-configs-pr.json
+
+ - name: Read auto configs PR configuration
+ id: read-config
+ run: |
+ if ! jq --exit-status '.profiles."${{ steps.parse.outputs.profile }}"' caller/config/auto-configs-pr.json; then
+ echo "::error::Profile ${{ steps.parse.outputs.profile }} does not exist in config/auto-configs-pr.json at ref ${{ steps.pr.outputs.ref }}"
+ exit 1
+ fi
+
+ configs=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs | keys' caller/config/auto-configs-pr.json)
+ configs_formatted=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs | keys | join(" ")' caller/config/auto-configs-pr.json)
+ configs_repo=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs_repo' caller/config/auto-configs-pr.json)
+
+ echo "$configs"
+ echo "$configs_formatted"
+ echo "$configs_repo"
+
+ echo "configs=$configs" >> $GITHUB_OUTPUT
+ echo "configs-formatted=$configs_formatted" >> $GITHUB_OUTPUT
+ echo "configs-repo=$configs_repo" >> $GITHUB_OUTPUT
+
+ update-configs:
+ name: Update
+ # Open PRs in the callers configs repository with the updated prerelease build
+ runs-on: ubuntu-latest
+ needs:
+ - setup
+ strategy:
+ fail-fast: false
+ matrix:
+ config: ${{ fromJson(needs.setup.outputs.configs) }}
+ env:
+ GH_TOKEN: ${{ secrets.configs-repo-token }}
+ CONFIG: ${{ matrix.config }}
+ PR_URL_ARTIFACT_GLOB: pr-urls-*
+ PR_URL_ARTIFACT_NAME: pr-urls-${{ matrix.config }}
+ outputs:
+ pr-url-artifact-glob: ${{ env.PR_URL_ARTIFACT_GLOB }}
+ # Within which contains the URL of the opened PR
+ # pr-url: ${{ steps.open-pr.outputs.pr-url }}
+ steps:
+ - name: Checkout caller repository
+ uses: actions/checkout@v4
+ with:
+ path: caller
+ ref: ${{ needs.setup.outputs.pr-branch }}
+
+ - name: Checkout caller configs repository
+ uses: actions/checkout@v4
+ with:
+ repository: ${{ needs.setup.outputs.configs-repo }}
+ path: caller-configs
+ fetch-depth: 0
+ token: ${{ secrets.configs-repo-token }}
+
+ - name: Checkout build-cd
+ uses: actions/checkout@v4
+ with:
+ repository: access-nri/build-cd
+ path: build-cd
+ ref: v7
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+ cache: pip
+
+ - name: Install Dependencies
+ run: pip install -q -q -r build-cd/scripts/model_config_manifest/requirements.txt
+
+ - name: Import Commit-Signing Key
+ uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4 # v6.1.0
+ with:
+ gpg_private_key: ${{ secrets.commit-gpg-private-key }}
+ passphrase: ${{ secrets.commit-gpg-passphrase }}
+ git_config_global: true
+ git_committer_name: ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }}
+ git_committer_email: ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }}
+ git_user_signingkey: true
+ git_commit_gpgsign: true
+ git_tag_gpgsign: true
+
+ - name: Get previous deployments in PR
+ id: previous-deployments
+ uses: access-nri/build-cd/.github/actions/get-pr-deployments@v7
+ with:
+ pr: ${{ github.event.issue.number }}
+ token: ${{ github.token }}
+
+ - name: Open PR
+ id: open-pr
+ env:
+ DEPLOYMENT_IDENTIFIER: ${{ needs.setup.outputs.root-sbd }}/pr${{ github.event.issue.number }}-${{ steps.previous-deployments.outputs.deployments }}
+ run: |
+ pr_branch_name="auto/pr${{ github.event.issue.number }}-${{ steps.previous-deployments.outputs.deployments }}/${{ env.CONFIG }}"
+
+ echo "Creating PR branch $pr_branch_name from ${{ env.CONFIG }}..."
+ git -C caller-configs checkout ${{ env.CONFIG }}
+
+ if git -C caller-configs ls-remote --exit-code --heads origin $pr_branch_name; then
+ # TODO: Feature: Append to the existing PR instead of continuing
+ echo "::error::Branch $pr_branch_name already exists for ${{ env.CONFIG }}, so a gh PR can't be created from it"
+ exit 1
+ fi
+
+ git -C caller-configs checkout -b $pr_branch_name
+
+ if [ ! -f caller-configs/config.yaml ]; then
+ echo "::error::File config.yaml not found in ${{ needs.setup.outputs.configs-repo }} on branch ${{ env.CONFIG }}, cannot automatically open configs PRs"
+ exit 1
+ fi
+
+ echo "Updating config.yaml in $pr_branch_name to use ${{ env.DEPLOYMENT_IDENTIFIER }}..."
+
+ # FIXME: This is Gadi-specific
+ # --manifest is still relative to the current working directory, unaffected by the custom PYTHONPATH
+ PYTHONPATH=build-cd python3 -m scripts.model_config_manifest.prerelease_update \
+ --manifest caller-configs/config.yaml \
+ --deployment-target Gadi \
+ --root-sbd ${{ needs.setup.outputs.root-sbd }} \
+ --module ${{ env.DEPLOYMENT_IDENTIFIER }}
+
+ echo "Committing and pushing changes..."
+ git -C caller-configs diff
+ git -C caller-configs commit -am "Auto update to use ${{ env.DEPLOYMENT_IDENTIFIER }} as part of ${{ env.RUN_URL }}"
+ git -C caller-configs push --set-upstream origin $pr_branch_name
+
+ echo "Opening PR for branch $pr_branch_name in repo ${{ needs.setup.outputs.configs-repo }}..."
+ pr_url=$(gh pr create \
+ --draft \
+ --repo "${{ needs.setup.outputs.configs-repo }}" \
+ --title "Auto update ${{ env.CONFIG }} to use ${{ env.DEPLOYMENT_IDENTIFIER }}" \
+ --body "Auto-generated PR to update ${{ env.CONFIG }} to use ${{ env.DEPLOYMENT_IDENTIFIER }} from ${{ github.repository }}#${{ github.event.issue.number }}, see ${{ env.RUN_URL }}" \
+ --head "$pr_branch_name" \
+ --base "${{ env.CONFIG }}"
+ )
+
+ echo "::notice::Opened PR: $pr_url"
+ echo "pr_url=$pr_url" >> $GITHUB_OUTPUT
+
+ - name: Determine if repro check required
+ id: repro-check
+ run: |
+ require_repro_check=$(jq --compact-output --raw-output \
+ --arg config "${{ env.CONFIG }}" \
+ '.profiles."${{ needs.setup.outputs.profile }}".configs.["${{ env.CONFIG }}"].checks.repro' \
+ caller/config/auto-configs-pr.json
+ )
+ echo "Repro check required: $require_repro_check"
+ echo "required=$require_repro_check" >> $GITHUB_OUTPUT
+
+ - name: Repro check
+ if: steps.repro-check.outputs.required == 'true'
+ run: |
+ echo "::notice::The updated ${{ env.CONFIG }} configuration requested a repro check, commenting !test repro on ${{ steps.open-pr.outputs.pr_url }}"
+ gh pr comment ${{ steps.open-pr.outputs.pr_url }} --repo "${{ needs.setup.outputs.configs-repo }}" --body '!test repro'
+
+ - name: Set PR URL Artifact
+ run: |
+ jq --null-input --arg pr_url "${{ steps.open-pr.outputs.pr_url }}" '{pr_url: $pr_url}' > ./${{ env.PR_URL_ARTIFACT_NAME }}
+
+ - name: Upload PR URL Artifact
+ uses: actions/upload-artifact@v6
+ with:
+ name: ${{ env.PR_URL_ARTIFACT_NAME }}
+ path: ./${{ env.PR_URL_ARTIFACT_NAME }}
+ if-no-files-found: error
+
+ result:
+ name: Result
+ if: always()
+ needs:
+ - setup
+ - update-configs
+ runs-on: ubuntu-latest
+ env:
+ ARTIFACT_PATH: ./pr-urls
+ steps:
+ - name: Checkout build-cd
+ uses: actions/checkout@v4
+ with:
+ repository: access-nri/build-cd
+ path: build-cd
+
+ - name: Download PR URL Artifacts
+ uses: actions/download-artifact@v6
+ with:
+ pattern: ${{ needs.update-configs.outputs.pr-url-artifact-glob }}
+ merge-multiple: true
+ path: ${{ env.ARTIFACT_PATH }}
+
+ - name: Setup python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+ cache: pip
+
+ - name: Install Jinja2
+ run: python3 -m pip install jinja-cli==1.2.2
+
+ - name: Comment on caller PR
+ env:
+ GH_TOKEN: ${{ github.token }}
+ # Get all the PR URLs from the matrix jobs and combine them into a single space-separated string
+ # Then pass it to jinja to comment templating
+ run: |
+ if [ -d "${{ env.ARTIFACT_PATH }}" ]; then
+ pr_urls=$(jq --slurp --raw-output \
+ '[.[] | .pr_url] | join(" ")' \
+ ${{ env.ARTIFACT_PATH }}/${{ needs.update-configs.outputs.pr-url-artifact-glob }}
+ )
+ fi
+
+ echo "PR URLs passed to template: $pr_urls"
+
+ jinja \
+ --define model_configs_repo "${{ needs.setup.outputs.configs-repo }}" \
+ --define run_url "${{ env.RUN_URL }}" \
+ --define pr_urls "$pr_urls" \
+ --define profile "${{ needs.setup.outputs.profile }}" \
+ --define configs "${{ needs.setup.outputs.configs-formatted }}" \
+ --define error "${{ contains(needs.*.result, 'failure') }}" \
+ build-cd/scripts/jinja_template/templates/auto-prs-comment-body.md.j2 \
+ > templated.auto-prs-comment-body.md
+
+ gh pr comment ${{ github.event.issue.number }} --repo ${{ github.repository }} --body-file templated.auto-prs-comment-body.md
diff --git a/.github/workflows/ci-comment.yml b/.github/workflows/ci-comment.yml
index d566ecd1..64e1565f 100644
--- a/.github/workflows/ci-comment.yml
+++ b/.github/workflows/ci-comment.yml
@@ -73,7 +73,7 @@ jobs:
- name: Original version
id: original
- uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v6
+ uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7
- name: Setup
id: setup
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e9b50b00..aff66093 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -122,29 +122,20 @@ jobs:
echo "sha=$sha" >> $GITHUB_OUTPUT
echo "base=$base" >> $GITHUB_OUTPUT
+ - name: Get previous PR deployments
+ id: previous-deploys
+ uses: access-nri/build-cd/.github/actions/get-pr-deployments@v7
+ with:
+ pr: ${{ inputs.pr }}
+
- name: Branch metadata
id: prerelease
- # Essentially, count all the deployment entries that match the given branch, as well as
- # all the `!redeploy` comments, to get the next deployment number.
- # See https://docs.github.com/en/rest/deployments/deployments?apiVersion=2022-11-28#list-deployments
+ # Since the total number of deployments do not include the current deployment if it is a PR deployment,
+ # but !redeploy comments do, we need to increment the next deployment number by one if it is a PR deployment.
run: |
- # We --slurp the results because --paginate introduces potentially multiple array results
- pr_deployments=$(gh api \
- -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
- --paginate --slurp \
- /repos/${{ github.repository }}/deployments \
- | jq '[select(.[][].ref == "${{ steps.pr.outputs.head }}")] | length'
- )
- comment_deployments=$(gh pr view ${{ inputs.pr }} --repo ${{ github.repository }} \
- --json comments \
- --jq '[.comments[] | select(.body | startswith("!redeploy"))] | length'
- )
-
- # Since the number of $pr_deployments do not include the current deployment (yet),
- # but $comment_deployments do, we need to increment the next deployment number by one if it is a pr deployment.
next_deployment_is_pr_deployment=${{ github.event_name == 'pull_request' && '1' || '0' }}
- next_deployment_number=$((pr_deployments + comment_deployments + next_deployment_is_pr_deployment))
- echo "Next Deployment Number is $pr_deployments + $comment_deployments + $next_deployment_is_pr_deployment = $next_deployment_number"
+ next_deployment_number=$((${{ steps.previous-deploys.outputs.deployments}} + next_deployment_is_pr_deployment))
+ echo "Next Deployment Number is ${{ steps.previous-deploys.outputs.deployments }} + $next_deployment_is_pr_deployment = $next_deployment_number"
echo "next-deployment-number=$next_deployment_number" >> $GITHUB_OUTPUT
version="pr${{ inputs.pr }}-$next_deployment_number"
@@ -153,7 +144,7 @@ jobs:
- name: Generate Deployment Target Matrix
id: target
- uses: access-nri/build-cd/.github/actions/get-target-matrix@v6
+ uses: access-nri/build-cd/.github/actions/get-target-matrix@v7
with:
targets: ${{ vars.PRERELEASE_DEPLOYMENT_TARGETS }}
@@ -217,7 +208,7 @@ jobs:
matrix:
# Example: ['Gadi', 'Setonix', ...]
target: ${{ fromJson(needs.defaults.outputs.targets) }}
- uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v6
+ uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v7
with:
deployment-target: ${{ matrix.target }}
deployment-ref: ${{ needs.defaults.outputs.head-ref }}
diff --git a/.github/workflows/deploy-1-setup.yml b/.github/workflows/deploy-1-setup.yml
index 7822f71b..fc245245 100644
--- a/.github/workflows/deploy-1-setup.yml
+++ b/.github/workflows/deploy-1-setup.yml
@@ -177,14 +177,14 @@ jobs:
- name: Validate spack-packages version
id: spack-packages
- uses: access-nri/build-cd/.github/actions/validate-repo-version@v6
+ uses: access-nri/build-cd/.github/actions/validate-repo-version@v7
with:
repo-to-check: spack-packages
pr: ${{ inputs.deployment-ref }}
- name: Validate spack version
id: spack
- uses: access-nri/build-cd/.github/actions/validate-repo-version@v6
+ uses: access-nri/build-cd/.github/actions/validate-repo-version@v7
with:
repo-to-check: spack
pr: ${{ inputs.deployment-ref }}
@@ -215,7 +215,7 @@ jobs:
- name: Validate build-cd config/settings.json
id: settings
- uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v6
+ uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v7
with:
settings-path: ./cd/config/settings.json
target: ${{ inputs.deployment-target }}
@@ -250,7 +250,7 @@ jobs:
- name: Get current (${{ inputs.deployment-ref }}) root spec version
id: current
- uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v6
+ uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7
with:
spack-manifest-path: ${{ inputs.spack-manifest-path }}
@@ -289,7 +289,7 @@ jobs:
- name: Get base root spec version
id: base
if: inputs.deployment-type != 'Release' && steps.checkout-base-spack.outcome != 'failure'
- uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v6
+ uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7
with:
spack-manifest-path: ${{ inputs.spack-manifest-path }}
@@ -330,7 +330,7 @@ jobs:
needs:
- check-config # Verify configuration information is correct
- check-spack-yaml # Verify spack manifest information is correct
- uses: access-nri/build-cd/.github/workflows/deploy-2-start.yml@v6
+ uses: access-nri/build-cd/.github/workflows/deploy-2-start.yml@v7
with:
ref: ${{ inputs.deployment-ref }}
version: ${{ inputs.deployment-version }}
diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml
index 579d4cee..7bd80b9f 100644
--- a/.github/workflows/deploy-2-start.yml
+++ b/.github/workflows/deploy-2-start.yml
@@ -87,7 +87,7 @@ jobs:
- name: Get ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} Remote Paths
id: path
- uses: access-nri/build-cd/.github/actions/get-deploy-paths@v6
+ uses: access-nri/build-cd/.github/actions/get-deploy-paths@v7
with:
spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_LOCATION }}
spack-version: ${{ steps.versions.outputs.spack }}
@@ -96,7 +96,7 @@ jobs:
- name: Get manifest info
id: manifest
- uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v6
+ uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7
with:
spack-manifest-path: ${{ inputs.spack-manifest-path }}
diff --git a/.github/workflows/settings-1-update.yml b/.github/workflows/settings-1-update.yml
index c05a53f2..ba8fef01 100644
--- a/.github/workflows/settings-1-update.yml
+++ b/.github/workflows/settings-1-update.yml
@@ -53,7 +53,7 @@ jobs:
- name: Validate Deployment Settings
id: validate
- uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v6
+ uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v7
with:
settings-path: ${{ env.CONFIG_SETTINGS_PATH }}
target: ${{ matrix.target }}
@@ -146,7 +146,7 @@ jobs:
# - deployment-environment: Gadi
# type: Prerelease
# etc ...
- uses: access-nri/build-cd/.github/workflows/settings-2-deploy.yml@v6
+ uses: access-nri/build-cd/.github/workflows/settings-2-deploy.yml@v7
with:
deployment-environment: ${{ matrix.update.deployment-environment }}
spack-type: ${{ matrix.update.type }}
diff --git a/.github/workflows/undeploy-1-start.yml b/.github/workflows/undeploy-1-start.yml
index 4c61ef44..ba60108f 100644
--- a/.github/workflows/undeploy-1-start.yml
+++ b/.github/workflows/undeploy-1-start.yml
@@ -48,7 +48,7 @@ jobs:
- name: Get ${{ inputs.target }} Remote Paths
id: path
- uses: access-nri/build-cd/.github/actions/get-deploy-paths@v6
+ uses: access-nri/build-cd/.github/actions/get-deploy-paths@v7
with:
spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_LOCATION }}
spack-version: ${{ steps.versions.outputs.spack }}
diff --git a/scripts/jinja_template/templates/auto-prs-comment-body.md.j2 b/scripts/jinja_template/templates/auto-prs-comment-body.md.j2
new file mode 100644
index 00000000..262bff87
--- /dev/null
+++ b/scripts/jinja_template/templates/auto-prs-comment-body.md.j2
@@ -0,0 +1,35 @@
+:wrench: Opening Model Configuration PRs in `{{ model_configs_repo if model_configs_repo != '' else 'an unknown configs repository' }}`
+
+{% if error == 'true' %}
+:x: One or more errors occurred with the workflow
+{% endif %}
+
+
+Configurations Requested
+
+{% if configs %}
+Configurations requested from profile `{{ profile }}`:
+{% for config in configs.split() %}
+- `{{ config }}`
+{% endfor %}
+{% else %}
+No configs were requested.
+{% endif %}
+
+
+
+
+Pull Requests Opened
+
+{% if pr_urls %}
+The following PRs were opened:
+{% for pr_url in pr_urls.split() %}
+- {{ pr_url }}
+{% endfor %}
+{% else %}
+No PRs were opened
+{% endif %}
+
+
+
+More details can be found in the workflow run: {{ run_url }}
\ No newline at end of file
diff --git a/scripts/model_config_manifest/prerelease_update.py b/scripts/model_config_manifest/prerelease_update.py
new file mode 100644
index 00000000..11a8d958
--- /dev/null
+++ b/scripts/model_config_manifest/prerelease_update.py
@@ -0,0 +1,143 @@
+import argparse
+import re
+import sys
+import yaml
+
+# Essentially we are looking to do the following substitutions in yq:
+# yq -i '.modules.use += ["/g/data/vk83/prerelease/modules"]' config.yaml
+# yq -i '.modules.load |= map(sub("^${{ needs.setup.outputs.root-sbd }}/.*"; "${{ env.DEPLOYMENT_IDENTIFIER }}"))' config.yaml
+# yq -i '.manifest.reproduce.exe=false' config.yaml
+
+def update_model_config_manifest(
+ manifest: dict[str, any],
+ deployment_target: str,
+ root_sbd: str,
+ module: str
+) -> dict[str, any]:
+ updated_manifest = update_modules_use_section(
+ manifest, deployment_target
+ )
+
+ updated_manifest = update_modules_load_section(
+ updated_manifest, root_sbd, module
+ )
+
+ updated_manifest = update_reproduce_exe_section(
+ updated_manifest
+ )
+
+ return updated_manifest
+
+def update_modules_use_section(
+ manifest: dict[str, any],
+ deployment_target: str
+) -> dict[str, any]:
+ """
+ Updates the modules.use section of the model config manifest to use the prerelease module path
+ """
+ manifest.setdefault("modules", {}).setdefault("use", [])
+
+ modules_use: list[str] = manifest["modules"]["use"]
+
+ match deployment_target:
+ case "Gadi":
+ prerelease_module_path = "/g/data/vk83/prerelease/modules"
+ case _:
+ raise ValueError(f"Unsupported deployment target: {deployment_target}")
+
+ if prerelease_module_path not in modules_use:
+ modules_use.append(prerelease_module_path)
+
+ manifest["modules"]["use"] = modules_use
+
+ return manifest
+
+def update_modules_load_section(
+ manifest: dict[str, any],
+ root_sbd: str,
+ prerelease_module: str
+) -> dict[str, any]:
+ """
+ Updates the modules.load section of the model config manifest to use the prerelease module name
+ """
+ manifest.setdefault("modules", {}).setdefault("load", [])
+
+ modules_load: list[str] = manifest["modules"]["load"]
+
+ # We remove all entries that start with the root_sbd to avoid conflicts with the existing release modules in the config.yaml
+ updated_modules_load = [prerelease_module] + [m for m in modules_load if not m.startswith(f"{root_sbd}/")]
+
+ print(f"When updating modules.load, removed entries starting with '{root_sbd}' and added '{prerelease_module}' giving: {updated_modules_load}")
+
+ manifest["modules"]["load"] = updated_modules_load
+
+ return manifest
+
+def update_reproduce_exe_section(
+ manifest: dict[str, any]
+) -> dict[str, any]:
+ """
+ Updates the manifest.reproduce.exe section of the model config manifest to be false for prerelease builds
+ """
+ manifest.setdefault("manifest", {}).setdefault("reproduce", {})
+
+ manifest["manifest"]["reproduce"]["exe"] = False
+
+ return manifest
+
+def parse_args(args: list[str]) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Script for updating model configuration repositories config.yaml to use prerelease builds."
+ )
+
+ ## Args dealing with inputs
+ parser.add_argument(
+ "--manifest",
+ type=str,
+ required=True,
+ help="Path to the spack manifest file to be injected with prerelease information",
+ )
+
+ parser.add_argument(
+ "--deployment-target",
+ type=str,
+ required=True,
+ help="Deployment target to be used for projections in the manifest",
+ )
+
+ parser.add_argument(
+ "--root-sbd",
+ type=str,
+ required=True,
+ help="Root Spack Bundle Definition to be used for module path updates",
+ )
+
+ parser.add_argument(
+ "--module",
+ type=str,
+ required=True,
+ help="Module name to be used for module load updates",
+ )
+
+ return parser.parse_args(args)
+
+
+def main():
+ args = parse_args(sys.argv[1:])
+
+ with open(args.manifest, "r") as f:
+ manifest = yaml.safe_load(f)
+
+ updated_manifest = update_model_config_manifest(
+ manifest,
+ args.deployment_target,
+ args.root_sbd,
+ args.module
+ )
+
+ with open(args.manifest, "w") as f:
+ yaml.safe_dump(updated_manifest, f)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/model_config_manifest/requirements.txt b/scripts/model_config_manifest/requirements.txt
new file mode 100644
index 00000000..8392d541
--- /dev/null
+++ b/scripts/model_config_manifest/requirements.txt
@@ -0,0 +1 @@
+PyYAML==6.0.2
diff --git a/tests/scripts/model_config_manifest/test_prerelease_update.py b/tests/scripts/model_config_manifest/test_prerelease_update.py
new file mode 100644
index 00000000..bd5832d1
--- /dev/null
+++ b/tests/scripts/model_config_manifest/test_prerelease_update.py
@@ -0,0 +1,141 @@
+import pytest
+import yaml
+
+from scripts.model_config_manifest.prerelease_update import (
+ update_model_config_manifest,
+ update_modules_use_section,
+ update_modules_load_section,
+ update_reproduce_exe_section,
+)
+
+#### Fixtures ####
+
+@pytest.fixture
+def valid_manifest() -> dict[str, any]:
+ return {
+ "jobname": "01deg_jra55_iaf",
+ "modules": {
+ "use": ["/g/data/vk83/modules"],
+ "load": ["access-om2/2025.12.000"],
+ },
+ "reproduce": {
+ "exe": True
+ }
+ }
+
+@pytest.fixture
+def vaild_similar_manifest() -> dict[str, any]:
+ return {
+ "jobname": "01deg_jra55_iaf",
+ "modules": {
+ "use": ["/g/data/vk83/prerelease/modules"],
+ "load": ["access-om2/2025.12.000", "other-module/1.0.0"],
+ },
+ "reproduce": {
+ "exe": False
+ }
+ }
+
+@pytest.fixture
+def empty_manifest() -> dict[str, any]:
+ return {}
+
+class TestPrereleaseUpdate:
+ ########################################
+ ## Testing update_modules_use_section ##
+ ########################################
+
+ def test_update_modules_use_section__valid(self, valid_manifest):
+ updated_manifest = update_modules_use_section(
+ valid_manifest,
+ deployment_target="Gadi"
+ )
+
+ assert updated_manifest["modules"]["use"] == [
+ "/g/data/vk83/modules",
+ "/g/data/vk83/prerelease/modules"
+ ]
+
+ def test_update_modules_use_section__vaild_similar(self, vaild_similar_manifest):
+ updated_manifest = update_modules_use_section(
+ vaild_similar_manifest,
+ deployment_target="Gadi"
+ )
+
+ assert updated_manifest["modules"]["use"] == [
+ "/g/data/vk83/prerelease/modules"
+ ]
+
+ def test_update_modules_use_section__vaild_empty(self, empty_manifest):
+ updated_manifest = update_modules_use_section(
+ empty_manifest,
+ deployment_target="Gadi"
+ )
+
+ assert updated_manifest["modules"]["use"] == [
+ "/g/data/vk83/prerelease/modules"
+ ]
+
+ #########################################
+ ## Testing update_modules_load_section ##
+ #########################################
+
+ def test_update_modules_load_section__valid(self, valid_manifest):
+ module = "access-om2/pr12-34"
+ updated_manifest = update_modules_load_section(
+ valid_manifest,
+ root_sbd="access-om2",
+ prerelease_module=module
+ )
+
+ assert updated_manifest["modules"]["load"] == [
+ module
+ ]
+
+ def test_update_modules_load_section__vaild_similar(self, vaild_similar_manifest):
+ module = "access-om2/pr12-34"
+ updated_manifest = update_modules_load_section(
+ vaild_similar_manifest,
+ root_sbd="access-om2",
+ prerelease_module=module
+ )
+
+ assert updated_manifest["modules"]["load"] == [
+ module,
+ "other-module/1.0.0"
+ ]
+
+ def test_update_modules_load_section__vaild_empty(self, empty_manifest):
+ module = "access-om2/pr12-34"
+ updated_manifest = update_modules_load_section(
+ empty_manifest,
+ root_sbd="access-om2",
+ prerelease_module=module
+ )
+
+ assert updated_manifest["modules"]["load"] == [module]
+
+ ##########################################
+ ## Testing update_reproduce_exe_section ##
+ ##########################################
+
+ def test_update_reproduce_exe_section__valid(self, valid_manifest):
+ updated_manifest = update_reproduce_exe_section(
+ valid_manifest
+ )
+
+ assert updated_manifest["manifest"]["reproduce"]["exe"] is False
+
+ def test_update_reproduce_exe_section__vaild_similar(self, vaild_similar_manifest):
+ updated_manifest = update_reproduce_exe_section(
+ vaild_similar_manifest
+ )
+
+ assert updated_manifest["manifest"]["reproduce"]["exe"] is False
+
+ def test_update_reproduce_exe_section__vaild_empty(self, empty_manifest):
+ updated_manifest = update_reproduce_exe_section(
+ empty_manifest
+ )
+
+ assert updated_manifest["manifest"]["reproduce"]["exe"] is False