deploy-2-start.yml

name: Deploy Start
on:
  workflow_call:
    inputs:
      ref:
        type: string
        required: true
        description: The git commit-ish ref where the spack manifest is located
      version:
        type: string
        required: true
        description: The version for the model being deployed
      env-name:
        type: string
        required: true
        description: The spack-env-compliant environment name for the model
      deployment-target:
        type: string
        required: true
        description: |
          The GitHub Environment, minus a Type.
          Combined with inputs.deployment-type to create the GitHub Environment name.
      deployment-type:
        type: string
        required: true
        description: |
          The type of deployment.
          Can be one of: Release, Prerelease.
          Combined with inputs.deployment-target to create the GitHub Environment name.
      expected-root-spec-name:
        type: string
        required: true
        description: |
          The expected root spec name that is defined at the repository level.
          May be different from the actual root spec name in the spack manifest.
      spack-manifest-path:
        type: string
        required: true
        description: The path to the spack manifest file, relative to the model deployment repository.
    outputs:
      root-spec-pkg-hash:
        value: ${{ jobs.deploy-to-environment.outputs.root-spec-pkg-hash }}
        description: The spack package long hash of the installed root spec
      modules-location:
        value: ${{ jobs.deploy-to-environment.outputs.modules-location }}
        description: The location of the modules directory on the deployment environment
      spack-location:
        value: ${{ jobs.deploy-to-environment.outputs.spack-location }}
        description: The location of the spack directory on the deployment environment
      general-metadata-artifact-glob:
        value: ${{ jobs.deploy-to-environment.outputs.metadata-artifact-name }}
        description: |
          General pattern for the artifact that contains the deployment metadata files for this invocation of the job.
          These files are of the form deploy-metadata.{{inputs.deployment-target}}
env:
  METADATA_PATH: /opt/metadata
  ARTIFACT_NAME: deploy-${{ inputs.expected-root-spec-name }}-metadata.${{ inputs.deployment-target }}
jobs:
  deploy-to-environment:
    name: Deploy to ${{ inputs.deployment-target }} ${{ inputs.deployment-type }}
    runs-on: ubuntu-latest
    environment: ${{ inputs.deployment-target }} ${{ inputs.deployment-type }}
    outputs:
      spack-location: ${{ steps.location.outputs.spack }}
      modules-location: ${{ steps.location.outputs.modules }}
      root-spec-pkg-hash: ${{ steps.metadata.outputs.root-spec-pkg-hash }}
      metadata-artifact-name: ${{ env.ARTIFACT_NAME }}
    steps:
      - name: Checkout ${{ github.repository }}
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}

      - name: Checkout build-cd
        uses: actions/checkout@v4
        with:
          repository: access-nri/build-cd
          path: build-cd

      - name: Get Versions From config/versions.json
        id: versions
        run: |
          echo "spack=$(jq --compact-output --raw-output '.spack' ./config/versions.json)" >> $GITHUB_OUTPUT
          echo "packages=$(jq --compact-output --raw-output '."access-spack-packages"' ./config/versions.json)" >> $GITHUB_OUTPUT
          echo "custom-scopes=$(jq --compact-output --raw-output '."custom-scopes" // [] | join(" ")' ./config/versions.json)" >> $GITHUB_OUTPUT

      - name: Get ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} Remote Paths
        id: path
        uses: access-nri/build-cd/.github/actions/get-deploy-paths@v8
        with:
          spack-installs-root-path: ${{ vars.SPACK_INSTALLS_ROOT_LOCATION }}
          spack-version: ${{ steps.versions.outputs.spack }}
          deployment-target: ${{ inputs.deployment-target }}
          deployment-type: ${{ inputs.deployment-type }}
          spack-environment: ${{ inputs.env-name }}

      - name: Setup SSH
        id: ssh
        uses: access-nri/actions/.github/actions/setup-ssh@main
        with:
          private-key: ${{ secrets.SSH_KEY }}
          hosts: |
            ${{ secrets.HOST }}
            ${{ secrets.HOST_DATA }}

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install Injection Requirements
        working-directory: build-cd
        run: python -m pip install -r scripts/spack_manifest/injection/requirements.txt

      - name: Spack Manifest - Get Provenance and Injection Packages
        id: config-packages
        run: |
          packages_for_injection=$(jq -cr '.provenance + .injection | join(",")' config/packages.json)
          packages_for_provenance=$(jq -cr '.provenance | join(" ")' config/packages.json)

          echo "Packages to be injected: $packages_for_injection"
          echo "Packages for provenance: $packages_for_provenance"

          echo "packages-for-injection=$packages_for_injection" >> $GITHUB_OUTPUT
          echo "packages-for-provenance=$packages_for_provenance" >> $GITHUB_OUTPUT

      - name: Spack Manifest - Modules Injection
        # Injects relevant module projections and includes into the spack manifest so users don't have to
        working-directory: build-cd
        run: |
          python -m scripts.spack_manifest.injection.modules \
            --manifest ../${{ inputs.spack-manifest-path }} \
            --packages ${{ steps.config-packages.outputs.packages-for-injection }} \
            --output ../${{ inputs.spack-manifest-path }}

      - name: Get SHA for access-spack-packages repository
        id: packages-ref
        uses: access-nri/actions/.github/actions/get-git-ref-info@main
        with:
          repository: ACCESS-NRI/access-spack-packages
          ref: ${{ steps.versions.outputs.packages }}

      - name: Spack Manifest - Prerelease Injection
        if: inputs.deployment-type == 'Prerelease'
        # Modifies the name of the prerelease modulefile to the pr<number>-<deployment number>` style.
        # Also adds a repos section that points to the prereleases specific access-spack-packages repo.
        working-directory: build-cd
        run: |
          python -m scripts.spack_manifest.injection.prerelease \
            --manifest ../${{ inputs.spack-manifest-path }} \
            --version ${{ inputs.version }} \
            --spack-packages-path ${{ steps.path.outputs.spack-environment }}/access-spack-packages \
            --spack-packages-version-sha ${{ steps.packages-ref.outputs.sha }} \
            --output ../${{ inputs.spack-manifest-path }}

      - name: Copy ${{ inputs.spack-manifest-path }}
        run: |
          rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
            ${{ inputs.spack-manifest-path }} \
            ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ vars.SPACK_YAML_LOCATION }}/${{ inputs.expected-root-spec-name }}-${{ github.run_id }}.spack.yaml

      - name: Fetch git sources for packages to build
        # Workaround for ACCESS-NRI/spack#2.
        # TODO: Remove once underlying spack issue is fixed
        run: |
          ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'

          if [ ! -d "${{ steps.path.outputs.root }}/git_repos" ]; then
            echo "::warning::Unable to find git_repos folder under ${{ steps.path.outputs.root }} - spack will attempt fetch instead"
            exit 0
          fi

          cd "${{ steps.path.outputs.root }}/git_repos" || exit 0

          for package_repo in *; do
            git -C "./$package_repo" fetch --verbose --update-head-ok origin "+refs/heads/*:refs/heads/*"
          done
          EOT

      - name: Deploy to ${{ inputs.deployment-target }} ${{ inputs.deployment-type }}
        id: deploy
        env:
          SCOPES: ${{ steps.versions.outputs.custom-scopes }}
          SCOPES_PATH: ${{ steps.path.outputs.spack-config }}/custom/cd
        # ssh into deployment environment, create and activate the env, install the spack manifest.
        run: |
          ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
          # Check that a suitable deployment location exists
          if [ ! -d "${{ steps.path.outputs.root }}" ]; then
            echo '::error::A deployment of spack does not exist in ${{ steps.path.outputs.root }} for ${{ inputs.deployment-target }} ${{ inputs.deployment-type}}'
            exit 1
          fi

          # Export vars.DEPLOYMENT_TARGET
          export DEPLOYMENT_TARGET="${{ vars.DEPLOYMENT_TARGET }}"
          echo "DEPLOYMENT_TARGET exported as $DEPLOYMENT_TARGET"

          # Enable spack
          . ${{ steps.path.outputs.spack-config }}/spack-enable.bash

          # Create and activate environment
          spack env activate ${{ inputs.env-name }} --create --envfile ${{ vars.SPACK_YAML_LOCATION }}/${{ inputs.expected-root-spec-name }}-${{ github.run_id }}.spack.yaml

          # Finally, install the spack manifest
          spack --debug \
            $(for s in ${{ env.SCOPES }}; do echo -n "--config-scope=${{ env.SCOPES_PATH }}/$s "; done) \
            install --fail-fast --fresh ${{ vars.SPACK_INSTALL_ADDITIONAL_ARGS }} || exit $?

          spack \
            $(for s in ${{ env.SCOPES }}; do echo -n "--config-scope=${{ env.SCOPES_PATH }}/$s "; done) \
            module tcl refresh -y
          EOT

      - name: Move spack logs
        if: always() && (steps.deploy.outcome == 'success' || steps.deploy.outcome == 'failure')
        run: |
          ssh ${{ secrets.USER}}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
          find . -maxdepth 1 -regex './spack-cc-.+' -exec mv {} ${{ steps.path.outputs.root }}/logs \;
          EOT

      - name: Export deployment target locations
        id: location
        run: |
          echo "spack=${{ steps.path.outputs.spack }}" >> $GITHUB_OUTPUT
          echo "modules=${{ vars.DEPLOYED_MODULES_DIR }}" >> $GITHUB_OUTPUT

      - name: Get metadata from ${{ inputs.deployment-target }} ${{ inputs.deployment-type}}
        id: metadata
        # Get the metadata from the spack environment regarding build database packages
        run: |
          ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
          # Export vars.DEPLOYMENT_TARGET
          export DEPLOYMENT_TARGET="${{ vars.DEPLOYMENT_TARGET }}"
          echo "DEPLOYMENT_TARGET exported as $DEPLOYMENT_TARGET"

          . ${{ steps.path.outputs.spack-config }}/spack-enable.bash
          spack env activate ${{ inputs.env-name }}

          spack find --paths > ${{ steps.path.outputs.spack-environment }}/spack.location

          # output the root-spec-pkg-hash into a file that is accessible outside the ssh session
          # FIXME: We currently pick the first root spec, in the unsupported cases there are multiple root specs. See #333
          root_spec_hashes=$(spack find --format '{hash}' ${{ inputs.expected-root-spec-name }})

          if [ $(wc -l <<<"$root_spec_hashes") -gt 1 ]; then
            echo "::warning::Multiple root specs found for ${{ inputs.expected-root-spec-name }}. Using the first one, as multiple root specs are currently unsupported."
          fi

          root_spec_hash=$(head -n 1 <<< "$root_spec_hashes")
          echo "$root_spec_hash" > ${{ steps.path.outputs.spack-environment }}/root-spec-pkg-hash.${{ github.run_id }}

          # Get the information associated with the packages for the build database
          jq -n '[]' > ${{ steps.path.outputs.spack-environment }}/build-db-pkgs.json

          for pkg in ${{ steps.config-packages.outputs.packages-for-provenance }}; do
            hash=$(spack find --format "{^${pkg}.hash}" /$root_spec_hash)
            version=$(spack find --format '{version}' /$hash)
            location=$(spack find --format '{prefix}' /$hash)
            pkg_repo_url=$(spack python -c "print(spack.repo.PATH.get_pkg_class('$pkg').git)")

            # We need to create a tmp output file as jq doesn't support in-place read/write
            jq \
              --arg p "$pkg" \
              --arg v "$version" \
              --arg h "$hash" \
              --arg l "$location" \
              --arg r "$pkg_repo_url" \
              '. += [{"name": $p, "version": $v, "hash": $h, "location": $l, "url": $r}]' \
              ${{ steps.path.outputs.spack-environment }}/build-db-pkgs.json > ${{ steps.path.outputs.spack-environment }}/build-db-pkgs.json.tmp

            mv -f ${{ steps.path.outputs.spack-environment }}/build-db-pkgs.json.tmp ${{ steps.path.outputs.spack-environment }}/build-db-pkgs.json
          done

          spack env deactivate
          EOT

          # Output root spec package hash from ssh session
          rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
            --remove-source-files \
            ${{ secrets.USER }}@${{ secrets.HOST }}:${{ steps.path.outputs.spack-environment }}/root-spec-pkg-hash.${{ github.run_id }} \
            ./root-spec-pkg-hash.${{ github.run_id }}

          echo "root-spec-pkg-hash=$(cat ./root-spec-pkg-hash.${{ github.run_id }})" >> $GITHUB_OUTPUT
          rm ./root-spec-pkg-hash.${{ github.run_id }}

      - name: Tar spack environment metadata
        # We want to attempt to tar up the .spack-env directory even if the install fails, so we can save inodes.
        if: always() && (steps.deploy.conclusion == 'success' || steps.deploy.conclusion == 'failure')
        run: |
          ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
          if [ -d "${{ steps.path.outputs.spack-environment }}/.spack-env" ]; then
            tar --remove-files --create --file ${{ steps.path.outputs.spack-environment }}/.spack-env.tar ${{ steps.path.outputs.spack-environment }}/.spack-env
          else
            echo "::warning::No .spack-env directory found in ${{ steps.path.outputs.spack-environment }}, not tarring. There may be an issue with the environment."
          fi
          EOT

      # Release
      - name: Get Release Metadata
        # Copy both the spack.* files and the build-db-pkgs.json file to the artifact
        run: |
          rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
            '${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ steps.path.outputs.spack-environment }}/spack.*' \
            '${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ steps.path.outputs.spack-environment }}/build-db-pkgs.json' \
            ./${{ env.ARTIFACT_NAME }}

          # Rename the files to include the deployment environment
          cd ./${{ env.ARTIFACT_NAME }}
          for file in *; do
            mv "$file" "${{ inputs.deployment-target }}.$file"
          done
          cd -

      - name: Upload Metadata Artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.ARTIFACT_NAME }}
          path: ./${{ env.ARTIFACT_NAME }}/*