diff --git a/.github/actions/get-pr-deployments/README.md b/.github/actions/get-pr-deployments/README.md new file mode 100644 index 00000000..1711f243 --- /dev/null +++ b/.github/actions/get-pr-deployments/README.md @@ -0,0 +1,33 @@ +# Get PR Deployments + +Action that returns the number of deployments on a given PR branch. + +## Inputs + +| Name | Description | Required | Default | Example | +| ---- | ----------- | -------- | ------- | ------- | +| `pr` | The pull request number to check for deployments of all kinds | `true` | N/A | `21` | +| `repository` | The repository to check for deployments | `false` | Value of `github.repository` | `"ACCESS-NRI/ACCESS-OM2"` | +| `token` | The GitHub token to use for API requests | `false` | Value of `github.token` | `"ghp_XXXX"` | + +## Outputs + +| Name | Description | Example | +| ---- | ----------- | ------- | +| `deployments` | The total number of deployments for the given PR (from commits and `!redeploy`s) | `24` | + +## Example + +```yaml +# ... +jobs: + deployments: + runs-on: ubuntu-latest + steps: + - id: pr-deploys + uses: access-nri/build-cd/.github/actions/get-pr-deployments@v6 + with: + pr: 12 + + - run: echo "There have been ${{ steps.pr-deploys.outputs.deployments }} total deployments in this PR, including both regular commit deployments and redeployments" +``` diff --git a/.github/actions/get-pr-deployments/action.yml b/.github/actions/get-pr-deployments/action.yml new file mode 100644 index 00000000..fc778bbd --- /dev/null +++ b/.github/actions/get-pr-deployments/action.yml @@ -0,0 +1,71 @@ +name: Get PR Deployments +description: Action that returns how many deployments were made in a PR (both from commits and !redeploys) +author: Tommy Gatti +inputs: + pr: + required: true + description: The pull request number to check for deployments of all kinds + repository: + required: false + default: ${{ github.repository }} + description: The repository to check for deployments + token: + required: false + default: ${{ github.token }} + description: The GitHub token to use for API requests +outputs: + deployments: + description: The total number of deployments for the given PR (from commits and !redeploys) + value: ${{ steps.total.outputs.number }} +runs: + using: composite + # Essentially, count all the deployment entries that match the given branch, as well as + # all the `!redeploy` comments, to get the next deployment number. + # See https://docs.github.com/en/rest/deployments/deployments?apiVersion=2022-11-28#list-deployments + steps: + - name: Get PR HEAD + id: pr + shell: bash + env: + GH_TOKEN: ${{ inputs.token }} + run: | + head=$(gh pr view ${{ inputs.pr }} --repo ${{ inputs.repository }} --json headRefName --jq .headRefName) + echo "HEAD of PR #${{ inputs.pr }} in ${{ inputs.repository }} is $head" + echo "head=$head" >> $GITHUB_OUTPUT + + - name: Get commit deployments for PR + id: commit + shell: bash + env: + GH_TOKEN: ${{ inputs.token }} + # We --slurp the results because --paginate introduces potentially multiple array results + run: | + deployments=$(gh api \ + -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \ + --paginate --slurp \ + /repos/${{ inputs.repository }}/deployments \ + | jq '[select(.[][].ref == "${{ steps.pr.outputs.head }}")] | length' + ) + echo "Found $deployments deployments for PR #${{ inputs.pr }} in ${{ inputs.repository }}" + echo "deployments=$deployments" >> $GITHUB_OUTPUT + + - name: Get !redeploys from comments + id: comment + shell: bash + env: + GH_TOKEN: ${{ inputs.token }} + run: | + redeployments=$(gh pr view ${{ inputs.pr }} --repo ${{ github.repository }} \ + --json comments \ + --jq '[.comments[] | select(.body | startswith("!redeploy"))] | length' + ) + echo "Found $redeployments redeploy comments for PR #${{ inputs.pr }} in ${{ inputs.repository }}" + echo "redeployments=$redeployments" >> $GITHUB_OUTPUT + + - name: Return total deployments + id: total + shell: bash + run: | + total=$(( ${{ steps.commit.outputs.deployments }} + ${{ steps.comment.outputs.redeployments }} )) + echo "Found $total deployments across commits (${{ steps.commit.outputs.deployments }}) and redeploys (${{ steps.comment.outputs.redeployments }}) for PR #${{ inputs.pr }} in ${{ inputs.repository }}" + echo "number=$total" >> $GITHUB_OUTPUT diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index efb379ef..28dcc7b5 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -91,7 +91,7 @@ jobs: - name: Generate Deployment Target Matrix id: target - uses: access-nri/build-cd/.github/actions/get-target-matrix@v6 + uses: access-nri/build-cd/.github/actions/get-target-matrix@v7 with: targets: ${{ vars.RELEASE_DEPLOYMENT_TARGETS }} @@ -108,7 +108,7 @@ jobs: with: repository: access-nri/build-cd - - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v6 + - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v7 with: settings-path: ./config/settings.json target: ${{ matrix.target }} @@ -126,7 +126,7 @@ jobs: - name: Get root spec ref id: tag - uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v6 + uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7 with: spack-manifest-path: ${{ inputs.spack-manifest-path }} @@ -180,7 +180,7 @@ jobs: strategy: matrix: target: ${{ fromJson(needs.defaults.outputs.targets) }} - uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v6 + uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v7 with: deployment-target: ${{ matrix.target }} deployment-ref: ${{ github.ref_name }} diff --git a/.github/workflows/ci-closed.yml b/.github/workflows/ci-closed.yml index afe51791..d96aedb3 100644 --- a/.github/workflows/ci-closed.yml +++ b/.github/workflows/ci-closed.yml @@ -38,7 +38,7 @@ jobs: - name: Generate Deployment Target Matrix id: target - uses: access-nri/build-cd/.github/actions/get-target-matrix@v6 + uses: access-nri/build-cd/.github/actions/get-target-matrix@v7 with: targets: ${{ vars.PRERELEASE_DEPLOYMENT_TARGETS }} @@ -50,7 +50,7 @@ jobs: matrix: target: ${{ fromJson(needs.setup.outputs.targets) }} fail-fast: false - uses: access-nri/build-cd/.github/workflows/undeploy-1-start.yml@v6 + uses: access-nri/build-cd/.github/workflows/undeploy-1-start.yml@v7 with: version-pattern: ${{ inputs.root-sbd }}-pr${{ github.event.pull_request.number }}-* target: ${{ matrix.target }} diff --git a/.github/workflows/ci-command-configs.yml b/.github/workflows/ci-command-configs.yml new file mode 100644 index 00000000..3f1d97cc --- /dev/null +++ b/.github/workflows/ci-command-configs.yml @@ -0,0 +1,359 @@ +name: Configs +on: + workflow_call: + inputs: + model: + type: string + required: true + description: The model that is being tested and deployed + root-sbd: + type: string + required: false + # The equivalent default is set in the defaults job below. + # default: ${{ inputs.model }} + description: | + The name of the root Spack Bundle Definition, if it is different from the model name. + This is often a package named similarly in ACCESS-NRI/spack-packages. + auto-configs-pr-schema-version: + type: string + required: true + description: The version of the auto-configs-pr schema to use for validation, from the ACCESS-NRI/schema repository. + secrets: + configs-repo-token: + required: true + description: | + The GitHub token to use for pushing and pull request creation in the *-configs repositories. + Requires pull-requests:write and contents:write permissions. + commit-gpg-private-key: + required: true + description: The private GPG key used to sign commits. + commit-gpg-passphrase: + required: true + description: The passphrase for the private GPG key used to sign commits. +env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_TOKEN: ${{ github.token }} + PYTHON_VERSION: '3.11' +jobs: + setup: + name: Setup + # Job is used to check permissions, parse the given !update-configs command, + # and read configuration information from the caller repository. + runs-on: ubuntu-latest + outputs: + # Value for the defaulted root-sbd if not provided + root-sbd: ${{ steps.defaults.outputs.root-sbd }} + # Caller PR branch ref + pr-branch: ${{ steps.pr.outputs.ref }} + # The profile to use from the MDRs config/auto-configs-pr.json + profile: ${{ steps.parse.outputs.profile }} + # The configs to update from the MDRs config/auto-configs-pr.json, in a GitHub Actions matrix-parseable JSON array + configs: ${{ steps.read-config.outputs.configs }} + # The configs to update, in a space-separated string + configs-formatted: ${{ steps.read-config.outputs.configs-formatted }} + # The repository containing the configs to update + configs-repo: ${{ steps.read-config.outputs.configs-repo }} + steps: + - name: Check commenter permissions + id: commenter + uses: access-nri/actions/.github/actions/commenter-permission-check@main + with: + # This means that commenters who use `!update-configs` must have at least `write` perms + # in the repository. + minimum-permission: write + + - name: React to Comment + uses: access-nri/actions/.github/actions/react-to-comment@main + with: + token: ${{ github.token }} + reaction: ${{ steps.commenter.outputs.has-permission == 'true' && 'rocket' || '-1' }} + + - name: Exit if no permission to run + if: steps.commenter.outputs.has-permission != 'true' + run: | + echo "::error::Commenter does not have permission to run this command, requires at least 'write' permission" + exit 1 + + - name: Parse command + id: parse + env: + USAGE: '!update-configs [profile=PROFILE]' + # We don't like ! event expansion here + shell: bash +H {0} + run: | + # Defaults + profile="default" + + # Parse comment into command args + comment='${{ github.event.comment.body }}' + read -r command <<< "$comment" + args=${command#!update-configs} + + # Processing args + # FIXME: Wouldn't handle positional args, if they were part of the command syntax + # If there isn't anything after !update-configs, args will be empty (or erroneously spaces, which we strip) + if [ -n "${args// }" ]; then + for arg in $args; do + case $arg in + profile=*) profile="${arg#profile=}" ;; + *) echo "::error::Unknown argument '$arg'. Usage: $USAGE"; exit 1;; + esac + done + fi + + # Output processed args + echo "profile=$profile" >> $GITHUB_OUTPUT + + - name: Set defaults + id: defaults + run: | + if [[ "${{ inputs.root-sbd }}" == "" ]]; then + echo "root-sbd=${{ inputs.model }}" >> $GITHUB_OUTPUT + else + echo "root-sbd=${{ inputs.root-sbd }}" >> $GITHUB_OUTPUT + fi + + - name: Get caller repository ref + id: pr + # We need to find the PR ref of the caller repository explicitly, as this workflow trigger (issue_comment) is in the context of the default branch + run: echo "ref=$(gh pr view ${{ github.event.issue.number }} --repo ${{ github.repository }} --json headRefName --jq .headRefName)" >> $GITHUB_OUTPUT + + - name: Checkout caller repository + uses: actions/checkout@v4 + with: + ref: ${{ steps.pr.outputs.ref }} + path: caller + + - name: Validate auto configs PR configuration + uses: access-nri/schema/.github/actions/validate-with-schema@main + with: + schema-version: ${{ inputs.auto-configs-pr-schema-version }} + schema-location: au.org.access-nri/model/deployment/config/auto-configs-pr + data-location: caller/config/auto-configs-pr.json + + - name: Read auto configs PR configuration + id: read-config + run: | + if ! jq --exit-status '.profiles."${{ steps.parse.outputs.profile }}"' caller/config/auto-configs-pr.json; then + echo "::error::Profile ${{ steps.parse.outputs.profile }} does not exist in config/auto-configs-pr.json at ref ${{ steps.pr.outputs.ref }}" + exit 1 + fi + + configs=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs | keys' caller/config/auto-configs-pr.json) + configs_formatted=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs | keys | join(" ")' caller/config/auto-configs-pr.json) + configs_repo=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs_repo' caller/config/auto-configs-pr.json) + + echo "$configs" + echo "$configs_formatted" + echo "$configs_repo" + + echo "configs=$configs" >> $GITHUB_OUTPUT + echo "configs-formatted=$configs_formatted" >> $GITHUB_OUTPUT + echo "configs-repo=$configs_repo" >> $GITHUB_OUTPUT + + update-configs: + name: Update + # Open PRs in the callers configs repository with the updated prerelease build + runs-on: ubuntu-latest + needs: + - setup + strategy: + fail-fast: false + matrix: + config: ${{ fromJson(needs.setup.outputs.configs) }} + env: + GH_TOKEN: ${{ secrets.configs-repo-token }} + CONFIG: ${{ matrix.config }} + PR_URL_ARTIFACT_GLOB: pr-urls-* + PR_URL_ARTIFACT_NAME: pr-urls-${{ matrix.config }} + outputs: + pr-url-artifact-glob: ${{ env.PR_URL_ARTIFACT_GLOB }} + # Within which contains the URL of the opened PR + # pr-url: ${{ steps.open-pr.outputs.pr-url }} + steps: + - name: Checkout caller repository + uses: actions/checkout@v4 + with: + path: caller + ref: ${{ needs.setup.outputs.pr-branch }} + + - name: Checkout caller configs repository + uses: actions/checkout@v4 + with: + repository: ${{ needs.setup.outputs.configs-repo }} + path: caller-configs + fetch-depth: 0 + token: ${{ secrets.configs-repo-token }} + + - name: Checkout build-cd + uses: actions/checkout@v4 + with: + repository: access-nri/build-cd + path: build-cd + ref: v7 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + + - name: Install Dependencies + run: pip install -q -q -r build-cd/scripts/model_config_manifest/requirements.txt + + - name: Import Commit-Signing Key + uses: crazy-max/ghaction-import-gpg@01dd5d3ca463c7f10f7f4f7b4f177225ac661ee4 # v6.1.0 + with: + gpg_private_key: ${{ secrets.commit-gpg-private-key }} + passphrase: ${{ secrets.commit-gpg-passphrase }} + git_config_global: true + git_committer_name: ${{ vars.GH_ACTIONS_BOT_GIT_USER_NAME }} + git_committer_email: ${{ vars.GH_ACTIONS_BOT_GIT_USER_EMAIL }} + git_user_signingkey: true + git_commit_gpgsign: true + git_tag_gpgsign: true + + - name: Get previous deployments in PR + id: previous-deployments + uses: access-nri/build-cd/.github/actions/get-pr-deployments@v7 + with: + pr: ${{ github.event.issue.number }} + token: ${{ github.token }} + + - name: Open PR + id: open-pr + env: + DEPLOYMENT_IDENTIFIER: ${{ needs.setup.outputs.root-sbd }}/pr${{ github.event.issue.number }}-${{ steps.previous-deployments.outputs.deployments }} + run: | + pr_branch_name="auto/pr${{ github.event.issue.number }}-${{ steps.previous-deployments.outputs.deployments }}/${{ env.CONFIG }}" + + echo "Creating PR branch $pr_branch_name from ${{ env.CONFIG }}..." + git -C caller-configs checkout ${{ env.CONFIG }} + + if git -C caller-configs ls-remote --exit-code --heads origin $pr_branch_name; then + # TODO: Feature: Append to the existing PR instead of continuing + echo "::error::Branch $pr_branch_name already exists for ${{ env.CONFIG }}, so a gh PR can't be created from it" + exit 1 + fi + + git -C caller-configs checkout -b $pr_branch_name + + if [ ! -f caller-configs/config.yaml ]; then + echo "::error::File config.yaml not found in ${{ needs.setup.outputs.configs-repo }} on branch ${{ env.CONFIG }}, cannot automatically open configs PRs" + exit 1 + fi + + echo "Updating config.yaml in $pr_branch_name to use ${{ env.DEPLOYMENT_IDENTIFIER }}..." + + # FIXME: This is Gadi-specific + # --manifest is still relative to the current working directory, unaffected by the custom PYTHONPATH + PYTHONPATH=build-cd python3 -m scripts.model_config_manifest.prerelease_update \ + --manifest caller-configs/config.yaml \ + --deployment-target Gadi \ + --root-sbd ${{ needs.setup.outputs.root-sbd }} \ + --module ${{ env.DEPLOYMENT_IDENTIFIER }} + + echo "Committing and pushing changes..." + git -C caller-configs diff + git -C caller-configs commit -am "Auto update to use ${{ env.DEPLOYMENT_IDENTIFIER }} as part of ${{ env.RUN_URL }}" + git -C caller-configs push --set-upstream origin $pr_branch_name + + echo "Opening PR for branch $pr_branch_name in repo ${{ needs.setup.outputs.configs-repo }}..." + pr_url=$(gh pr create \ + --draft \ + --repo "${{ needs.setup.outputs.configs-repo }}" \ + --title "Auto update ${{ env.CONFIG }} to use ${{ env.DEPLOYMENT_IDENTIFIER }}" \ + --body "Auto-generated PR to update ${{ env.CONFIG }} to use ${{ env.DEPLOYMENT_IDENTIFIER }} from ${{ github.repository }}#${{ github.event.issue.number }}, see ${{ env.RUN_URL }}" \ + --head "$pr_branch_name" \ + --base "${{ env.CONFIG }}" + ) + + echo "::notice::Opened PR: $pr_url" + echo "pr_url=$pr_url" >> $GITHUB_OUTPUT + + - name: Determine if repro check required + id: repro-check + run: | + require_repro_check=$(jq --compact-output --raw-output \ + --arg config "${{ env.CONFIG }}" \ + '.profiles."${{ needs.setup.outputs.profile }}".configs.["${{ env.CONFIG }}"].checks.repro' \ + caller/config/auto-configs-pr.json + ) + echo "Repro check required: $require_repro_check" + echo "required=$require_repro_check" >> $GITHUB_OUTPUT + + - name: Repro check + if: steps.repro-check.outputs.required == 'true' + run: | + echo "::notice::The updated ${{ env.CONFIG }} configuration requested a repro check, commenting !test repro on ${{ steps.open-pr.outputs.pr_url }}" + gh pr comment ${{ steps.open-pr.outputs.pr_url }} --repo "${{ needs.setup.outputs.configs-repo }}" --body '!test repro' + + - name: Set PR URL Artifact + run: | + jq --null-input --arg pr_url "${{ steps.open-pr.outputs.pr_url }}" '{pr_url: $pr_url}' > ./${{ env.PR_URL_ARTIFACT_NAME }} + + - name: Upload PR URL Artifact + uses: actions/upload-artifact@v6 + with: + name: ${{ env.PR_URL_ARTIFACT_NAME }} + path: ./${{ env.PR_URL_ARTIFACT_NAME }} + if-no-files-found: error + + result: + name: Result + if: always() + needs: + - setup + - update-configs + runs-on: ubuntu-latest + env: + ARTIFACT_PATH: ./pr-urls + steps: + - name: Checkout build-cd + uses: actions/checkout@v4 + with: + repository: access-nri/build-cd + path: build-cd + + - name: Download PR URL Artifacts + uses: actions/download-artifact@v6 + with: + pattern: ${{ needs.update-configs.outputs.pr-url-artifact-glob }} + merge-multiple: true + path: ${{ env.ARTIFACT_PATH }} + + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + + - name: Install Jinja2 + run: python3 -m pip install jinja-cli==1.2.2 + + - name: Comment on caller PR + env: + GH_TOKEN: ${{ github.token }} + # Get all the PR URLs from the matrix jobs and combine them into a single space-separated string + # Then pass it to jinja to comment templating + run: | + if [ -d "${{ env.ARTIFACT_PATH }}" ]; then + pr_urls=$(jq --slurp --raw-output \ + '[.[] | .pr_url] | join(" ")' \ + ${{ env.ARTIFACT_PATH }}/${{ needs.update-configs.outputs.pr-url-artifact-glob }} + ) + fi + + echo "PR URLs passed to template: $pr_urls" + + jinja \ + --define model_configs_repo "${{ needs.setup.outputs.configs-repo }}" \ + --define run_url "${{ env.RUN_URL }}" \ + --define pr_urls "$pr_urls" \ + --define profile "${{ needs.setup.outputs.profile }}" \ + --define configs "${{ needs.setup.outputs.configs-formatted }}" \ + --define error "${{ contains(needs.*.result, 'failure') }}" \ + build-cd/scripts/jinja_template/templates/auto-prs-comment-body.md.j2 \ + > templated.auto-prs-comment-body.md + + gh pr comment ${{ github.event.issue.number }} --repo ${{ github.repository }} --body-file templated.auto-prs-comment-body.md diff --git a/.github/workflows/ci-comment.yml b/.github/workflows/ci-comment.yml index d566ecd1..64e1565f 100644 --- a/.github/workflows/ci-comment.yml +++ b/.github/workflows/ci-comment.yml @@ -73,7 +73,7 @@ jobs: - name: Original version id: original - uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v6 + uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7 - name: Setup id: setup diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9b50b00..aff66093 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,29 +122,20 @@ jobs: echo "sha=$sha" >> $GITHUB_OUTPUT echo "base=$base" >> $GITHUB_OUTPUT + - name: Get previous PR deployments + id: previous-deploys + uses: access-nri/build-cd/.github/actions/get-pr-deployments@v7 + with: + pr: ${{ inputs.pr }} + - name: Branch metadata id: prerelease - # Essentially, count all the deployment entries that match the given branch, as well as - # all the `!redeploy` comments, to get the next deployment number. - # See https://docs.github.com/en/rest/deployments/deployments?apiVersion=2022-11-28#list-deployments + # Since the total number of deployments do not include the current deployment if it is a PR deployment, + # but !redeploy comments do, we need to increment the next deployment number by one if it is a PR deployment. run: | - # We --slurp the results because --paginate introduces potentially multiple array results - pr_deployments=$(gh api \ - -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \ - --paginate --slurp \ - /repos/${{ github.repository }}/deployments \ - | jq '[select(.[][].ref == "${{ steps.pr.outputs.head }}")] | length' - ) - comment_deployments=$(gh pr view ${{ inputs.pr }} --repo ${{ github.repository }} \ - --json comments \ - --jq '[.comments[] | select(.body | startswith("!redeploy"))] | length' - ) - - # Since the number of $pr_deployments do not include the current deployment (yet), - # but $comment_deployments do, we need to increment the next deployment number by one if it is a pr deployment. next_deployment_is_pr_deployment=${{ github.event_name == 'pull_request' && '1' || '0' }} - next_deployment_number=$((pr_deployments + comment_deployments + next_deployment_is_pr_deployment)) - echo "Next Deployment Number is $pr_deployments + $comment_deployments + $next_deployment_is_pr_deployment = $next_deployment_number" + next_deployment_number=$((${{ steps.previous-deploys.outputs.deployments}} + next_deployment_is_pr_deployment)) + echo "Next Deployment Number is ${{ steps.previous-deploys.outputs.deployments }} + $next_deployment_is_pr_deployment = $next_deployment_number" echo "next-deployment-number=$next_deployment_number" >> $GITHUB_OUTPUT version="pr${{ inputs.pr }}-$next_deployment_number" @@ -153,7 +144,7 @@ jobs: - name: Generate Deployment Target Matrix id: target - uses: access-nri/build-cd/.github/actions/get-target-matrix@v6 + uses: access-nri/build-cd/.github/actions/get-target-matrix@v7 with: targets: ${{ vars.PRERELEASE_DEPLOYMENT_TARGETS }} @@ -217,7 +208,7 @@ jobs: matrix: # Example: ['Gadi', 'Setonix', ...] target: ${{ fromJson(needs.defaults.outputs.targets) }} - uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v6 + uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v7 with: deployment-target: ${{ matrix.target }} deployment-ref: ${{ needs.defaults.outputs.head-ref }} diff --git a/.github/workflows/deploy-1-setup.yml b/.github/workflows/deploy-1-setup.yml index 7822f71b..fc245245 100644 --- a/.github/workflows/deploy-1-setup.yml +++ b/.github/workflows/deploy-1-setup.yml @@ -177,14 +177,14 @@ jobs: - name: Validate spack-packages version id: spack-packages - uses: access-nri/build-cd/.github/actions/validate-repo-version@v6 + uses: access-nri/build-cd/.github/actions/validate-repo-version@v7 with: repo-to-check: spack-packages pr: ${{ inputs.deployment-ref }} - name: Validate spack version id: spack - uses: access-nri/build-cd/.github/actions/validate-repo-version@v6 + uses: access-nri/build-cd/.github/actions/validate-repo-version@v7 with: repo-to-check: spack pr: ${{ inputs.deployment-ref }} @@ -215,7 +215,7 @@ jobs: - name: Validate build-cd config/settings.json id: settings - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v6 + uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v7 with: settings-path: ./cd/config/settings.json target: ${{ inputs.deployment-target }} @@ -250,7 +250,7 @@ jobs: - name: Get current (${{ inputs.deployment-ref }}) root spec version id: current - uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v6 + uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7 with: spack-manifest-path: ${{ inputs.spack-manifest-path }} @@ -289,7 +289,7 @@ jobs: - name: Get base root spec version id: base if: inputs.deployment-type != 'Release' && steps.checkout-base-spack.outcome != 'failure' - uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v6 + uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7 with: spack-manifest-path: ${{ inputs.spack-manifest-path }} @@ -330,7 +330,7 @@ jobs: needs: - check-config # Verify configuration information is correct - check-spack-yaml # Verify spack manifest information is correct - uses: access-nri/build-cd/.github/workflows/deploy-2-start.yml@v6 + uses: access-nri/build-cd/.github/workflows/deploy-2-start.yml@v7 with: ref: ${{ inputs.deployment-ref }} version: ${{ inputs.deployment-version }} diff --git a/.github/workflows/deploy-2-start.yml b/.github/workflows/deploy-2-start.yml index 579d4cee..7bd80b9f 100644 --- a/.github/workflows/deploy-2-start.yml +++ b/.github/workflows/deploy-2-start.yml @@ -87,7 +87,7 @@ jobs: - name: Get ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} Remote Paths id: path - uses: access-nri/build-cd/.github/actions/get-deploy-paths@v6 + uses: access-nri/build-cd/.github/actions/get-deploy-paths@v7 with: spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_LOCATION }} spack-version: ${{ steps.versions.outputs.spack }} @@ -96,7 +96,7 @@ jobs: - name: Get manifest info id: manifest - uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v6 + uses: access-nri/build-cd/.github/actions/get-spack-root-spec@v7 with: spack-manifest-path: ${{ inputs.spack-manifest-path }} diff --git a/.github/workflows/settings-1-update.yml b/.github/workflows/settings-1-update.yml index c05a53f2..ba8fef01 100644 --- a/.github/workflows/settings-1-update.yml +++ b/.github/workflows/settings-1-update.yml @@ -53,7 +53,7 @@ jobs: - name: Validate Deployment Settings id: validate - uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v6 + uses: access-nri/build-cd/.github/actions/validate-deployment-settings@v7 with: settings-path: ${{ env.CONFIG_SETTINGS_PATH }} target: ${{ matrix.target }} @@ -146,7 +146,7 @@ jobs: # - deployment-environment: Gadi # type: Prerelease # etc ... - uses: access-nri/build-cd/.github/workflows/settings-2-deploy.yml@v6 + uses: access-nri/build-cd/.github/workflows/settings-2-deploy.yml@v7 with: deployment-environment: ${{ matrix.update.deployment-environment }} spack-type: ${{ matrix.update.type }} diff --git a/.github/workflows/undeploy-1-start.yml b/.github/workflows/undeploy-1-start.yml index 4c61ef44..ba60108f 100644 --- a/.github/workflows/undeploy-1-start.yml +++ b/.github/workflows/undeploy-1-start.yml @@ -48,7 +48,7 @@ jobs: - name: Get ${{ inputs.target }} Remote Paths id: path - uses: access-nri/build-cd/.github/actions/get-deploy-paths@v6 + uses: access-nri/build-cd/.github/actions/get-deploy-paths@v7 with: spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_LOCATION }} spack-version: ${{ steps.versions.outputs.spack }} diff --git a/scripts/jinja_template/templates/auto-prs-comment-body.md.j2 b/scripts/jinja_template/templates/auto-prs-comment-body.md.j2 new file mode 100644 index 00000000..262bff87 --- /dev/null +++ b/scripts/jinja_template/templates/auto-prs-comment-body.md.j2 @@ -0,0 +1,35 @@ +:wrench: Opening Model Configuration PRs in `{{ model_configs_repo if model_configs_repo != '' else 'an unknown configs repository' }}` + +{% if error == 'true' %} +:x: One or more errors occurred with the workflow +{% endif %} + +
+Configurations Requested + +{% if configs %} +Configurations requested from profile `{{ profile }}`: +{% for config in configs.split() %} +- `{{ config }}` +{% endfor %} +{% else %} +No configs were requested. +{% endif %} + +
+ +
+Pull Requests Opened + +{% if pr_urls %} +The following PRs were opened: +{% for pr_url in pr_urls.split() %} +- {{ pr_url }} +{% endfor %} +{% else %} +No PRs were opened +{% endif %} + +
+ +More details can be found in the workflow run: {{ run_url }} \ No newline at end of file diff --git a/scripts/model_config_manifest/prerelease_update.py b/scripts/model_config_manifest/prerelease_update.py new file mode 100644 index 00000000..11a8d958 --- /dev/null +++ b/scripts/model_config_manifest/prerelease_update.py @@ -0,0 +1,143 @@ +import argparse +import re +import sys +import yaml + +# Essentially we are looking to do the following substitutions in yq: +# yq -i '.modules.use += ["/g/data/vk83/prerelease/modules"]' config.yaml +# yq -i '.modules.load |= map(sub("^${{ needs.setup.outputs.root-sbd }}/.*"; "${{ env.DEPLOYMENT_IDENTIFIER }}"))' config.yaml +# yq -i '.manifest.reproduce.exe=false' config.yaml + +def update_model_config_manifest( + manifest: dict[str, any], + deployment_target: str, + root_sbd: str, + module: str +) -> dict[str, any]: + updated_manifest = update_modules_use_section( + manifest, deployment_target + ) + + updated_manifest = update_modules_load_section( + updated_manifest, root_sbd, module + ) + + updated_manifest = update_reproduce_exe_section( + updated_manifest + ) + + return updated_manifest + +def update_modules_use_section( + manifest: dict[str, any], + deployment_target: str +) -> dict[str, any]: + """ + Updates the modules.use section of the model config manifest to use the prerelease module path + """ + manifest.setdefault("modules", {}).setdefault("use", []) + + modules_use: list[str] = manifest["modules"]["use"] + + match deployment_target: + case "Gadi": + prerelease_module_path = "/g/data/vk83/prerelease/modules" + case _: + raise ValueError(f"Unsupported deployment target: {deployment_target}") + + if prerelease_module_path not in modules_use: + modules_use.append(prerelease_module_path) + + manifest["modules"]["use"] = modules_use + + return manifest + +def update_modules_load_section( + manifest: dict[str, any], + root_sbd: str, + prerelease_module: str +) -> dict[str, any]: + """ + Updates the modules.load section of the model config manifest to use the prerelease module name + """ + manifest.setdefault("modules", {}).setdefault("load", []) + + modules_load: list[str] = manifest["modules"]["load"] + + # We remove all entries that start with the root_sbd to avoid conflicts with the existing release modules in the config.yaml + updated_modules_load = [prerelease_module] + [m for m in modules_load if not m.startswith(f"{root_sbd}/")] + + print(f"When updating modules.load, removed entries starting with '{root_sbd}' and added '{prerelease_module}' giving: {updated_modules_load}") + + manifest["modules"]["load"] = updated_modules_load + + return manifest + +def update_reproduce_exe_section( + manifest: dict[str, any] +) -> dict[str, any]: + """ + Updates the manifest.reproduce.exe section of the model config manifest to be false for prerelease builds + """ + manifest.setdefault("manifest", {}).setdefault("reproduce", {}) + + manifest["manifest"]["reproduce"]["exe"] = False + + return manifest + +def parse_args(args: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Script for updating model configuration repositories config.yaml to use prerelease builds." + ) + + ## Args dealing with inputs + parser.add_argument( + "--manifest", + type=str, + required=True, + help="Path to the spack manifest file to be injected with prerelease information", + ) + + parser.add_argument( + "--deployment-target", + type=str, + required=True, + help="Deployment target to be used for projections in the manifest", + ) + + parser.add_argument( + "--root-sbd", + type=str, + required=True, + help="Root Spack Bundle Definition to be used for module path updates", + ) + + parser.add_argument( + "--module", + type=str, + required=True, + help="Module name to be used for module load updates", + ) + + return parser.parse_args(args) + + +def main(): + args = parse_args(sys.argv[1:]) + + with open(args.manifest, "r") as f: + manifest = yaml.safe_load(f) + + updated_manifest = update_model_config_manifest( + manifest, + args.deployment_target, + args.root_sbd, + args.module + ) + + with open(args.manifest, "w") as f: + yaml.safe_dump(updated_manifest, f) + + +if __name__ == "__main__": + main() diff --git a/scripts/model_config_manifest/requirements.txt b/scripts/model_config_manifest/requirements.txt new file mode 100644 index 00000000..8392d541 --- /dev/null +++ b/scripts/model_config_manifest/requirements.txt @@ -0,0 +1 @@ +PyYAML==6.0.2 diff --git a/tests/scripts/model_config_manifest/test_prerelease_update.py b/tests/scripts/model_config_manifest/test_prerelease_update.py new file mode 100644 index 00000000..bd5832d1 --- /dev/null +++ b/tests/scripts/model_config_manifest/test_prerelease_update.py @@ -0,0 +1,141 @@ +import pytest +import yaml + +from scripts.model_config_manifest.prerelease_update import ( + update_model_config_manifest, + update_modules_use_section, + update_modules_load_section, + update_reproduce_exe_section, +) + +#### Fixtures #### + +@pytest.fixture +def valid_manifest() -> dict[str, any]: + return { + "jobname": "01deg_jra55_iaf", + "modules": { + "use": ["/g/data/vk83/modules"], + "load": ["access-om2/2025.12.000"], + }, + "reproduce": { + "exe": True + } + } + +@pytest.fixture +def vaild_similar_manifest() -> dict[str, any]: + return { + "jobname": "01deg_jra55_iaf", + "modules": { + "use": ["/g/data/vk83/prerelease/modules"], + "load": ["access-om2/2025.12.000", "other-module/1.0.0"], + }, + "reproduce": { + "exe": False + } + } + +@pytest.fixture +def empty_manifest() -> dict[str, any]: + return {} + +class TestPrereleaseUpdate: + ######################################## + ## Testing update_modules_use_section ## + ######################################## + + def test_update_modules_use_section__valid(self, valid_manifest): + updated_manifest = update_modules_use_section( + valid_manifest, + deployment_target="Gadi" + ) + + assert updated_manifest["modules"]["use"] == [ + "/g/data/vk83/modules", + "/g/data/vk83/prerelease/modules" + ] + + def test_update_modules_use_section__vaild_similar(self, vaild_similar_manifest): + updated_manifest = update_modules_use_section( + vaild_similar_manifest, + deployment_target="Gadi" + ) + + assert updated_manifest["modules"]["use"] == [ + "/g/data/vk83/prerelease/modules" + ] + + def test_update_modules_use_section__vaild_empty(self, empty_manifest): + updated_manifest = update_modules_use_section( + empty_manifest, + deployment_target="Gadi" + ) + + assert updated_manifest["modules"]["use"] == [ + "/g/data/vk83/prerelease/modules" + ] + + ######################################### + ## Testing update_modules_load_section ## + ######################################### + + def test_update_modules_load_section__valid(self, valid_manifest): + module = "access-om2/pr12-34" + updated_manifest = update_modules_load_section( + valid_manifest, + root_sbd="access-om2", + prerelease_module=module + ) + + assert updated_manifest["modules"]["load"] == [ + module + ] + + def test_update_modules_load_section__vaild_similar(self, vaild_similar_manifest): + module = "access-om2/pr12-34" + updated_manifest = update_modules_load_section( + vaild_similar_manifest, + root_sbd="access-om2", + prerelease_module=module + ) + + assert updated_manifest["modules"]["load"] == [ + module, + "other-module/1.0.0" + ] + + def test_update_modules_load_section__vaild_empty(self, empty_manifest): + module = "access-om2/pr12-34" + updated_manifest = update_modules_load_section( + empty_manifest, + root_sbd="access-om2", + prerelease_module=module + ) + + assert updated_manifest["modules"]["load"] == [module] + + ########################################## + ## Testing update_reproduce_exe_section ## + ########################################## + + def test_update_reproduce_exe_section__valid(self, valid_manifest): + updated_manifest = update_reproduce_exe_section( + valid_manifest + ) + + assert updated_manifest["manifest"]["reproduce"]["exe"] is False + + def test_update_reproduce_exe_section__vaild_similar(self, vaild_similar_manifest): + updated_manifest = update_reproduce_exe_section( + vaild_similar_manifest + ) + + assert updated_manifest["manifest"]["reproduce"]["exe"] is False + + def test_update_reproduce_exe_section__vaild_empty(self, empty_manifest): + updated_manifest = update_reproduce_exe_section( + empty_manifest + ) + + assert updated_manifest["manifest"]["reproduce"]["exe"] is False