Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 21 additions & 30 deletions .github/workflows/cd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,7 @@ on:
model:
type: string
required: true
description: The model that is being tested and deployed
root-sbd:
type: string
required: false
# default: The ${{ inputs.model }} above
description: |
The name of the root Spack Bundle Definition, if it is different from the model name.
This is often a package named similarly in ACCESS-NRI/spack-packages.
description: The human-readable model name that is being tested and deployed
spack-manifest-path:
type: string
required: false
Expand Down Expand Up @@ -72,22 +65,20 @@ env:
METADATA_PATH: /opt/metadata
OUTPUTS_PATH: /opt/outputs
jobs:
defaults:
name: Set Defaults
# Unfortunately, you can't set a dynamic default value based on `inputs` yet
init:
name: Initialize Deployment
runs-on: ubuntu-latest
outputs:
root-sbd: ${{ steps.root-sbd.outputs.default }}
deployment-name: ${{ steps.manifest.outputs.deployment-name }}
targets: ${{ steps.target.outputs.valid-targets }}
steps:
- name: root-sbd
id: root-sbd
run: |
if [ -z "${{ inputs.root-sbd }}" ]; then
echo "default=${{ inputs.model }}" >> $GITHUB_OUTPUT
else
echo "default=${{ inputs.root-sbd }}" >> $GITHUB_OUTPUT
fi
- uses: actions/checkout@v4

- name: Get manifest
id: manifest
uses: access-nri/build-cd/.github.actions/get-spack-manifest@v8
with:
spack-manifest-path: ${{ inputs.spack-manifest-path }}

- name: Generate Deployment Target Matrix
id: target
Expand All @@ -99,10 +90,10 @@ jobs:
name: Verify Deployment Settings
runs-on: ubuntu-latest
needs:
- defaults
- init
strategy:
matrix:
target: ${{ fromJson(needs.defaults.outputs.targets) }}
target: ${{ fromJson(needs.init.outputs.targets) }}
steps:
- uses: actions/checkout@v4
with:
Expand Down Expand Up @@ -163,26 +154,26 @@ jobs:
deploy-release:
name: Deploy Release
needs:
- defaults
- init
- get-deployment-version
- push-tag
# Pushing tags is optional if we are doing a non-model deployment
if: >-
always() &&
needs.defaults.result == 'success' &&
needs.init.result == 'success' &&
needs.get-deployment-version.result == 'success' &&
(needs.push-tag.result == 'success' || needs.push-tag.result == 'skipped')
strategy:
matrix:
target: ${{ fromJson(needs.defaults.outputs.targets) }}
target: ${{ fromJson(needs.init.outputs.targets) }}
uses: access-nri/build-cd/.github/workflows/deploy-1-setup.yml@v8
with:
deployment-target: ${{ matrix.target }}
deployment-ref: ${{ github.ref_name }}
deployment-type: Release
deployment-version: ${{ needs.get-deployment-version.outputs.version }}
spack-manifest-path: ${{ inputs.spack-manifest-path }}
expected-root-spec-name: ${{ needs.defaults.outputs.root-sbd }}
expected-root-spec-name: ${{ needs.init.outputs.deployment-name }}
spack-manifest-schema-path: ${{ inputs.spack-manifest-schema-path }}
spack-manifest-schema-version: ${{ inputs.spack-manifest-schema-version }}
config-versions-schema-version: ${{ inputs.config-versions-schema-version }}
Expand All @@ -198,7 +189,7 @@ jobs:
name: Create Release
if: inputs.tag-deployment
needs:
- defaults
- init
- get-deployment-version
- push-tag
- deploy-release
Expand Down Expand Up @@ -241,7 +232,7 @@ jobs:
env:
J2_MODEL: ${{ inputs.model }}
J2_VERSION: ${{ needs.get-deployment-version.outputs.version }}
J2_ROOT_SBD: ${{ needs.defaults.outputs.root-sbd }}
J2_ROOT_SBD: ${{ needs.init.outputs.deployment-name }}
run: |
python -m scripts.jinja_template.render_deployment_info \
--template scripts/jinja_template/templates/release-body.md.j2 \
Expand All @@ -265,7 +256,7 @@ jobs:
name: Build DB Metadata Upload
if: inputs.tag-deployment && inputs.upload-to-build-db
needs:
- defaults
- init
- deploy-release
- release
runs-on: ubuntu-latest
Expand Down Expand Up @@ -308,7 +299,7 @@ jobs:
run: |
python -m scripts.release_provenance.tracking_services_data \
--repository ${{ github.repository }} \
--root-spec ${{ needs.defaults.outputs.root-sbd }} \
--root-spec ${{ needs.init.outputs.deployment-name }} \
--deployment-outputs ${{ env.OUTPUTS_PATH }} \
--metadata-outputs ${{ env.METADATA_PATH }} \
--upload
21 changes: 15 additions & 6 deletions .github/workflows/ci-closed.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: PR Closed Cleanup
run-name: ${{ inputs.model }} PR Closed Cleanup
run-name: PR Closed Cleanup
# Remove prereleases that were part of a closed PR, so we save space
# on our deployment targets. If needed, one can still get the
# spack.yaml as part of the closed PR and revive it themselves.
Expand All @@ -9,10 +9,10 @@ run-name: ${{ inputs.model }} PR Closed Cleanup
on:
workflow_call:
inputs:
root-sbd:
spack-manifest-path:
type: string
required: true
description: The model package name that is going to be removed
required: false
default: ./spack.yaml
# Callers usually have the trigger:
# pull_request:
# types:
Expand All @@ -31,10 +31,19 @@ jobs:
version-pattern: ${{ steps.version.outputs.pattern }}
targets: ${{ steps.target.outputs.valid-targets }}
steps:
- name: Get data from ${{ github.repository }}
uses: actions/checkout@v4

- name: Get Deployment Name
id: manifest
uses: access-nri/build-cd/.github/actions/get-spack-manifest@v8
with:
spack-manifest-path: ${{ inputs.spack-manifest-path }}

- name: Get Version Pattern
id: version
# For example, `access-om3-pr12-*`
run: echo "pattern=${{ inputs.root-sbd }}-pr${{ github.event.pull_request.number }}-*" >> $GITHUB_OUTPUT
run: echo "pattern=${{ steps.manifest.outputs.deployment-name }}-pr${{ github.event.pull_request.number }}-*" >> $GITHUB_OUTPUT

- name: Generate Deployment Target Matrix
id: target
Expand All @@ -52,7 +61,7 @@ jobs:
fail-fast: false
uses: access-nri/build-cd/.github/workflows/undeploy-1-start.yml@v8
with:
version-pattern: ${{ inputs.root-sbd }}-pr${{ github.event.pull_request.number }}-*
version-pattern: ${{ needs.setup.outputs.version-pattern }}
target: ${{ matrix.target }}
type: Prerelease
secrets: inherit
47 changes: 20 additions & 27 deletions .github/workflows/ci-command-configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,15 @@ name: Configs
on:
workflow_call:
inputs:
spack-manifest-path:
type: string
required: false
default: spack.yaml
description: Path to the spack manifest in the model deployment repository
model:
type: string
required: true
description: The model that is being tested and deployed
root-sbd:
type: string
required: false
# The equivalent default is set in the defaults job below.
# default: ${{ inputs.model }}
description: |
The name of the root Spack Bundle Definition, if it is different from the model name.
This is often a package named similarly in ACCESS-NRI/spack-packages.
auto-configs-pr-schema-version:
type: string
required: true
Expand Down Expand Up @@ -41,8 +38,8 @@ jobs:
# and read configuration information from the caller repository.
runs-on: ubuntu-latest
outputs:
# Value for the defaulted root-sbd if not provided
root-sbd: ${{ steps.defaults.outputs.root-sbd }}
# Deployment Name for the spack manifest - corresponds to the root spec
deployment-name: ${{ steps.manifest.outputs.deployment-name }}
# Caller PR branch ref
pr-branch: ${{ steps.pr.outputs.ref }}
# The profile to use from the MDRs config/auto-configs-pr.json
Expand Down Expand Up @@ -104,15 +101,6 @@ jobs:
# Output processed args
echo "profile=$profile" >> $GITHUB_OUTPUT

- name: Set defaults
id: defaults
run: |
if [[ "${{ inputs.root-sbd }}" == "" ]]; then
echo "root-sbd=${{ inputs.model }}" >> $GITHUB_OUTPUT
else
echo "root-sbd=${{ inputs.root-sbd }}" >> $GITHUB_OUTPUT
fi

- name: Get caller repository ref
id: pr
# We need to find the PR ref of the caller repository explicitly, as this workflow trigger (issue_comment) is in the context of the default branch
Expand All @@ -122,26 +110,25 @@ jobs:
uses: actions/checkout@v4
with:
ref: ${{ steps.pr.outputs.ref }}
path: caller

- name: Validate auto configs PR configuration
uses: access-nri/schema/.github/actions/validate-with-schema@main
with:
schema-version: ${{ inputs.auto-configs-pr-schema-version }}
schema-location: au.org.access-nri/model/deployment/config/auto-configs-pr
data-location: caller/config/auto-configs-pr.json
data-location: config/auto-configs-pr.json

- name: Read auto configs PR configuration
id: read-config
run: |
if ! jq --exit-status '.profiles."${{ steps.parse.outputs.profile }}"' caller/config/auto-configs-pr.json; then
if ! jq --exit-status '.profiles."${{ steps.parse.outputs.profile }}"' config/auto-configs-pr.json; then
echo "::error::Profile ${{ steps.parse.outputs.profile }} does not exist in config/auto-configs-pr.json at ref ${{ steps.pr.outputs.ref }}"
exit 1
fi

configs=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs | keys' caller/config/auto-configs-pr.json)
configs_formatted=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs | keys | join(" ")' caller/config/auto-configs-pr.json)
configs_repo=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs_repo' caller/config/auto-configs-pr.json)
configs=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs | keys' config/auto-configs-pr.json)
configs_formatted=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs | keys | join(" ")' config/auto-configs-pr.json)
configs_repo=$(jq -cr '.profiles."${{ steps.parse.outputs.profile }}".configs_repo' config/auto-configs-pr.json)

echo "$configs"
echo "$configs_formatted"
Expand All @@ -151,6 +138,12 @@ jobs:
echo "configs-formatted=$configs_formatted" >> $GITHUB_OUTPUT
echo "configs-repo=$configs_repo" >> $GITHUB_OUTPUT

- name: Get caller deployment name
id: manifest
uses: access-nri/build-cd/.github/actions/get-spack-manifest@v8
with:
spack-manifest-path: ${{ inputs.spack-manifest-path }}

update-configs:
name: Update
# Open PRs in the callers configs repository with the updated prerelease build
Expand Down Expand Up @@ -223,7 +216,7 @@ jobs:
- name: Open PR
id: open-pr
env:
DEPLOYMENT_IDENTIFIER: ${{ needs.setup.outputs.root-sbd }}/pr${{ github.event.issue.number }}-${{ steps.previous-deployments.outputs.deployments }}
DEPLOYMENT_IDENTIFIER: ${{ needs.setup.outputs.deployment-name }}/pr${{ github.event.issue.number }}-${{ steps.previous-deployments.outputs.deployments }}
run: |
pr_branch_name="auto/pr${{ github.event.issue.number }}-${{ steps.previous-deployments.outputs.deployments }}/${{ env.CONFIG }}"

Expand All @@ -250,7 +243,7 @@ jobs:
PYTHONPATH=build-cd python3 -m scripts.model_config_manifest.prerelease_update \
--manifest caller-configs/config.yaml \
--deployment-target Gadi \
--root-sbd ${{ needs.setup.outputs.root-sbd }} \
--root-sbd ${{ needs.setup.outputs.deployment-name }} \
--module ${{ env.DEPLOYMENT_IDENTIFIER }}

echo "Committing and pushing changes..."
Expand Down
42 changes: 10 additions & 32 deletions .github/workflows/ci-comment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,18 @@ run-name: ${{ inputs.model }} Comment Command
# permissions.contents:write
# permissions.pull-requests:write

# FIXME: We don't support any !bump from a spack manifest that is not in the default ./spack.yaml location.

on:
workflow_call:
inputs:
model:
type: string
required: true
description: The model that is being tested and deployed
root-sbd:
spack-manifest-path:
type: string
required: false
# The equivalent default is set in the defaults job below.
# default: ${{ inputs.model }}
description: |
The name of the root Spack Bundle Definition, if it is different from the model name.
This is often a package named similarly in ACCESS-NRI/spack-packages.
default: spack.yaml
description: Path the the spack manifest in the caller
# Callers usually have the trigger:
# issue_comment:
# types:
Expand All @@ -30,34 +25,15 @@ on:
env:
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
jobs:
defaults:
name: Set Defaults
# Unfortunately, you can't set a dynamic default value based on `inputs` yet.
runs-on: ubuntu-latest
outputs:
root-sbd: ${{ steps.root-sbd.outputs.default }}
steps:
- name: root-sbd default
id: root-sbd
run: |
if [[ "${{ inputs.root-sbd }}" == "" ]]; then
echo "default=${{ inputs.model }}" >> $GITHUB_OUTPUT
else
echo "default=${{ inputs.root-sbd }}" >> $GITHUB_OUTPUT
fi

bump-version:
name: Bump spack.yaml
if: github.event.issue.pull_request && startsWith(github.event.comment.body, '!bump')
needs:
- defaults
runs-on: ubuntu-latest
permissions:
pull-requests: write
contents: write
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SPACK_YAML_MODEL_PROJECTION_YQ: .spack.modules.default.tcl.projections.${{ needs.defaults.outputs.root-sbd }}
steps:
- name: Get branch from issue
id: issue
Expand All @@ -74,6 +50,8 @@ jobs:
- name: Original version
id: original
uses: access-nri/build-cd/.github/actions/get-spack-manifest@v8
with:
spack-manifest-path: ${{ inputs.spack-manifest-path }}

- name: Setup
id: setup
Expand Down Expand Up @@ -132,20 +110,20 @@ jobs:

- name: Update, Commit and Push the Bump
run: |
yq -i '.spack.definitions[]._version[0] = "${{ steps.bump.outputs.after }}"' spack.yaml
git add spack.yaml
git commit -m "spack.yaml: Updated deployment version from ${{ steps.original.outputs.deployment-version }} to ${{ steps.bump.outputs.after }}"
yq -i '.spack.definitions[]._version[0] = "${{ steps.bump.outputs.after }}"' ${{ inputs.spack-manifest-path }}
git add ${{ inputs.spack-manifest-path }}
git commit -m "${{ inputs.spack-manifest-path }}: Updated deployment version from ${{ steps.original.outputs.deployment-version }} to ${{ steps.bump.outputs.after }}"
git push

- name: Success Notifier
uses: access-nri/actions/.github/actions/pr-comment@main
with:
comment: |
:white_check_mark: Version bumped from `${{ steps.original.outputs.deployment-version }}` to `${{ steps.bump.outputs.after }}` :white_check_mark:
:white_check_mark: ${{ inputs.spack-manifest-path }} version bumped from `${{ steps.original.outputs.deployment-version }}` to `${{ steps.bump.outputs.after }}` :white_check_mark:

- name: Failure Notifier
if: failure()
uses: access-nri/actions/.github/actions/pr-comment@main
with:
comment: |
:x: Failed to bump version or commit changes, see ${{ env.RUN_URL }} :x:
:x: Failed to bump version or commit changes to ${{ inputs.spack-manifest-path }}, see ${{ env.RUN_URL }} :x:
Loading