Metadata Creation Uses spack --config-scope Scopes (#357)

* Update get_data_from_deployment script to take in `spack --config-scope` argument, updated usages

* Bounds checking for non-custom-scope builds

* Update print statement

* Raise spacks own error for failing to find a configuration scope

Author: CodeGat

.github/workflows/deploy-2-start.yml

@@ -82,7 +82,9 @@ jobs:
         run: |
           echo "spack=$(jq --compact-output --raw-output '.spack' ./config/versions.json)" >> $GITHUB_OUTPUT
           echo "packages=$(jq --compact-output --raw-output '."access-spack-packages"' ./config/versions.json)" >> $GITHUB_OUTPUT
-          echo "custom-scopes=$(jq --compact-output --raw-output '."custom-scopes" // [] | join(" ")' ./config/versions.json)" >> $GITHUB_OUTPUT
+          echo "custom-scopes-space-separated=$(jq --compact-output --raw-output '."custom-scopes" // [] | join(" ")' ./config/versions.json)" >> $GITHUB_OUTPUT
+          echo "custom-scopes-comma-separated=$(jq --compact-output --raw-output '."custom-scopes" // [] | join(",")' ./config/versions.json)" >> $GITHUB_OUTPUT
+
 
       - name: Get ${{ inputs.deployment-target }} ${{ inputs.deployment-type }} Remote Paths
         id: path
@@ -181,7 +183,7 @@ jobs:
       - name: Deploy to ${{ inputs.deployment-target }} ${{ inputs.deployment-type }}
         id: deploy
         env:
-          SCOPES: ${{ steps.versions.outputs.custom-scopes }}
+          SCOPES: ${{ steps.versions.outputs.custom-scopes-space-separated }}
           SCOPES_PATH: ${{ steps.path.outputs.spack-config }}/custom/cd
         # ssh into deployment environment, create and activate the env, install the spack manifest.
         run: |
@@ -243,6 +245,7 @@ jobs:
             --environment ${{ steps.path.outputs.spack-environment }} \
             --deployment-name ${{ inputs.expected-root-spec-name }} \
             --packages ${{ steps.config-packages.outputs.packages-for-provenance }} \
+            --config-scopes ${{ steps.versions.outputs.custom-scopes-comma-separated }} \
             --output ${{ steps.path.outputs.spack-environment }}
           EOT
 

scripts/release_provenance/get_data_from_deployment.py

@@ -9,16 +9,25 @@
 from typing import Any, List, Dict
 
 import spack.environment
+import spack.error
 import spack.cmd
+import spack.config
+import spack.paths
 import spack.spec
 import spack.repo
+import spack.main
 
 
 def main():
     args = parse_args(sys.argv[1:])
     packages: List[str] = args.packages.split(",")
+    config_scopes: List[str] = args.config_scopes.split(",") if args.config_scopes else []
     output_path = Path(args.output)
 
+    # Custom scopes added via spack --config-scope for install need to be added back here
+    # so we can find those packages!
+    add_custom_spack_config_scopes(config_scopes)
+
     # Activate the spack environment so we can get relevant specs for this deployment
     spack_env = activate_spack_environment(args.environment)
 
@@ -50,6 +59,26 @@ def main():
     with open(output_path / "build-db-pkgs.json", 'w') as f:
         json.dump(packages_metadata, f)
 
+def add_custom_spack_config_scopes(config_scopes: List[str]) -> None:
+    """
+    Adds paths to custom spack config scopes to the command_line scope so we can find binaries for
+    certain environments that use custom installation directories.
+
+    :param config_scopes: Names of custom scopes from spack-configs custom/cd directory.
+    :type config_scopes: List[str]
+    """
+    spack_config_custom_scopes_path: Path = Path(spack.paths.spack_root).parent / "spack-config" / "custom" / "cd"
+
+    config_scope_paths: List[str] = [str(spack_config_custom_scopes_path / s) for s in config_scopes]
+
+    print(f"Attempting to load custom scopes: {config_scope_paths}")
+
+    try:
+        spack.main.add_command_line_scopes(spack.config.CONFIG, config_scope_paths)
+    except spack.error.ConfigError:
+        print(f"Failed to find valid config scope in paths {config_scope_paths}.")
+        raise
+
 def activate_spack_environment(spack_env_path: str) -> spack.environment.Environment:
     spack_env = spack.environment.Environment(spack_env_path)
 
@@ -144,6 +173,13 @@ def parse_args(args: List[str]) -> argparse.Namespace:
         help="Comma-separated list of packages to extract build metadata for",
     )
 
+    parser.add_argument(
+        "--config-scopes",
+        type=str,
+        required=False,
+        help="Comma-separated list of custom spack config scopes defined in spack-configs custom/cd directory"
+    )
+
     ## Args dealing with outputs
     parser.add_argument(
         "--output",