vars to File-Based Special Packages (#305)

* Validate MDR config/packages.json

* Pass info from MDR config/packages.json to module injection script

* Update variable name to `vars.CONFIG_PACKAGES_SCHEMA_VERSION`

* Change cwd relative to build-cd rather than model for scripts

* Add output of packages to be injected/added to provenance DB

* Update prerelease projection to `ROOT_SPEC/prX-Y/{name}/VERSION`

* Update tests for new prerelease injection logic

Author: CodeGat

.github/workflows/deploy-1-setup.yml

@@ -148,6 +148,13 @@ jobs:
           schema-location: au.org.access-nri/model/deployment/config/versions
           data-location: ./model/config/versions.json
 
+      - name: Validate ${{ github.repository }} config/packages.json
+        uses: access-nri/schema/.github/actions/validate-with-schema@main
+        with:
+          schema-version: ${{ vars.CONFIG_PACKAGES_SCHEMA_VERSION }}
+          schema-location: au.org.access-nri/model/deployment/config/packages
+          data-location: ./model/config/packages.json
+
       - name: Validate spack-packages version
         id: spack-packages
         uses: access-nri/build-cd/.github/actions/validate-repo-version@v5

.github/workflows/deploy-2-start.yml

@@ -121,25 +121,28 @@ jobs:
 
       - name: Spack Manifest - Modules Injection
         # Injects relevant module projections and includes into the spack manifest so users don't have to
-        # Note the cd prefix to the module name, as this is the path we cloned build-cd into
         run: |
-          python -m cd.scripts.spack_manifest.injection.modules \
-            --manifest ${{ inputs.spack-manifest-path }} \
-            --packages "${{ vars.BUILD_DB_PACKAGES }}" \
-            --output ${{ inputs.spack-manifest-path }}
+          packages_for_injection=$(jq -cr '.provenance + .injection | join(" ")' config/packages.json)
+          echo "Packages to be injected: $packages_for_injection"
+
+          cd cd
+          python -m scripts.spack_manifest.injection.modules \
+            --manifest ../${{ inputs.spack-manifest-path }} \
+            --packages "$packages_for_injection" \
+            --output ../${{ inputs.spack-manifest-path }}
 
       - name: Spack Manifest - Prerelease Injection
         if: inputs.deployment-type == 'Prerelease'
         # Modifies the name of the prerelease modulefile to the pr<number>-<deployment number>` style.
         # Also removes the `@git.VERSION` specifier for Prereleases as it is a tag that doesn't yet exist.
         # Also adds a repos section that points to the prereleases specific spack-packages repo.
-        # Note the cd prefix to the module name, as this is the path we cloned build-cd into
+        working-directory: cd
         run: |
-          python -m cd.scripts.spack_manifest.injection.prerelease \
-            --manifest ${{ inputs.spack-manifest-path }} \
+          python -m scripts.spack_manifest.injection.prerelease \
+            --manifest ../${{ inputs.spack-manifest-path }} \
             --version ${{ inputs.version }} \
             --spack-packages-path ${{ steps.path.outputs.spack-packages }} \
-            --output ${{ inputs.spack-manifest-path }}
+            --output ../${{ inputs.spack-manifest-path }}
 
       - name: Copy ${{ inputs.spack-manifest-path }}
         run: |
@@ -252,7 +255,10 @@ jobs:
           # Get the information associated with the packages for the build database
           jq -n '[]' > ${{ steps.path.outputs.spack-environment }}/build-db-pkgs.json
 
-          for pkg in ${{ vars.BUILD_DB_PACKAGES }}; do
+          packages_for_provenance=$(jq -cr '.provenance | join(" ")' config/packages.json)
+          echo "Packages for provenance: $packages_for_provenance"
+
+          for pkg in $packages_for_provenance; do
             hash=$(spack find --format '{hash}' $pkg)
             version=$(spack find --format '{version}' $pkg)
             location=$(spack find --format '{prefix}' $pkg)

scripts/spack_manifest/injection/prerelease.py

@@ -69,9 +69,10 @@ def add_namespace_to_other_projection_versions(
         projections.pop(root_spec_name)
 
     for projection_name, projection_value in projections.items():
-        # Non-root-spec projections will be of the form {name}/prX-Y/VERSION, where VERSION is previously defined.
+        # Non-root-spec projections will be of the form ROOT_SPEC_NAME/prX-Y/{name}/VERSION, where VERSION is previously defined.
+        # For example, access-om2/pr12-13/mom5/main-{hash:7}
         new_projection_value = re.sub(
-            r"{name}/(.+)", rf"{{name}}/{version}/\1", projection_value
+            r"{name}/(.+)", rf"{root_spec_name}/{version}/{{name}}/\1", projection_value
         )
 
         print(

tests/scripts/spack_manifest/injection/outputs/expected.prerelease.spack.yaml

@@ -44,7 +44,7 @@ spack:
         - oasis3-mct
         projections:
           access-om2: '{name}/pr12-12'
-          mom5: '{name}/pr12-12/2023.11.09-{hash:7}'
+          mom5: access-om2/pr12-12/{name}/2023.11.09-{hash:7}
   repos::
   - /some/spack-packages
   - $spack/var/spack/repos/builtin

tests/scripts/spack_manifest/injection/test_prerelease.py

@@ -180,9 +180,9 @@ def test_add_namespace_to_other_projection_versions__valid(self):
         updated_manifest = add_namespace_to_other_projection_versions(manifest, root_spec_name, version)
 
         assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"]["access-om2"] == f"{{name}}/{version}"
-        assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"]["dependency1"] == f"{{name}}/{version}/2.0.0"
-        assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"]["dependency2"] == f"{{name}}/{version}/3.0.0-{{hash:7}}"
-        assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"]["dependency3"] == f"{{name}}/{version}/special/4.0.0-{{hash:7}}"
+        assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"]["dependency1"] == f"{root_spec_name}/{version}/{{name}}/2.0.0"
+        assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"]["dependency2"] == f"{root_spec_name}/{version}/{{name}}/3.0.0-{{hash:7}}"
+        assert updated_manifest["spack"]["modules"]["default"]["tcl"]["projections"]["dependency3"] == f"{root_spec_name}/{version}/{{name}}/special/4.0.0-{{hash:7}}"
 
     def test_add_namespace_to_other_projection_versions__no_projections_except_for_root(self):
         root_spec_name = "access-om2"