Add ci-at-desk, a script for locally running CI workflows (#24)

* Add ci-at-desk, a script for locally running CI workflows
Add `crash` as an extraction keyword in run-test-helper.py
Add `-ffile-prefix-map` argument to OpenUSD building (CI and documentation)

Author: maddyadams

.github/scripts/build-openusd.py

@@ -26,6 +26,13 @@
 from swiftusd_ci_common import *
 
 def install_cmake():
+    if which("cmake"):
+        print("CMake is already on PATH, will not reinstall")
+        return
+    else:
+        print(os.environ)
+        print("Couldn't find CMake, will install")
+
     print("Downloading CMake...")
     run(["curl", "-L", "https://github.com/Kitware/CMake/releases/download/v3.28.6/cmake-3.28.6-macos-universal.dmg",
         "--output", "CMake.dmg"],

.github/scripts/compute-openusd-build-cache-key.py

@@ -32,7 +32,10 @@
     # Turn branch names like `dev` into a hash commit to avoid incorrect cache key matches
     clone_openusd()
     rev_parsed_ref = run(["git", "rev-parse", Environment.GitRef.openusd], cwd=Environment.Path.openusd, logOutput=False).output[0]
-    build_flags = "".join(get_openusd_build_flags(Environment.TestCombination.target_platform))
+    # Don't include the build directory when forming the cache key
+    build_flags = get_openusd_build_flags(Environment.TestCombination.target_platform)
+    build_flags = [x for x in build_flags if not x.startswith("/")]
+    build_flags = "".join(build_flags)
 
     result = "  ".join([
         Environment.TestCombination.target_platform,
@@ -43,4 +46,7 @@
         openusd_patch_hash
     ])
 
-    printAndWrite(output=f"cache-key='{result}'")
+    # Cache keys cannot contain commas
+    result = result.replace(",", "")
+
+    printAndWrite(output=f"cache-key={result}")

.github/scripts/define-test-matrix.py

@@ -23,25 +23,44 @@
 import json
 import math
 
+def get_xcodebuild_destination(target_platform):
+    if target_platform == "macOS": return "platform=macOS,name=My Mac"
+    elif target_platform == "iOS": return Environment.TestCombination.at_desk_iOS_xcodebuild_destination
+    elif target_platform == "iOSSimulator": return "platform=iOS Simulator,name=iPhone 17 Pro"
+    elif target_platform == "visionOS": return Environment.TestCombination.at_desk_visionOS_xcodebuild_destination
+    elif target_platform == "visionOSSimulator": return "platform=visionOS Simulator,name=Apple Vision Pro (at 2732x2048)"
+    else:
+        print(f"Error: Unknown target '{target_platform}'")
+        exit(1)
+
 if __name__ == "__main__":
-    target_platforms = ["macOS", "iOSSimulator", "visionOSSimulator"]
+    target_platforms = ["macOS", "iOS", "iOSSimulator", "visionOS", "visionOSSimulator"]
     configs = ["Debug", "Release"]
     build_systems = ["xcodebuild-xcodeproj", "swiftbuild-SPM-Tests", "xcodebuild-SPM-Tests"]
 
     all_combinations = []
     for target_platform in target_platforms:
         for config in configs:
             for build_system in build_systems:
-
-                # if target_platform == "macOS" and build_system == "xcodebuild-xcodeproj":
-                #     # Disabled for now
-                #     continue
-
+                exclusivity_keys = []
+                
                 if build_system == "swiftbuild-SPM-Tests" and target_platform != "macOS":
                     # swiftbuild only supports macOS
                     continue
 
-                all_combinations.append({"target_platform" : target_platform, "config" : config, "build_system" : build_system})
+                if build_system == "xcodebuild-SPM-Tests" and target_platform in ["iOS", "visionOS"]:
+                    # xcodebuild on a Swift Package doesn't support physical iOS/visionOS devices
+                    continue
+
+                xcodebuild_destination = get_xcodebuild_destination(target_platform)
+                if xcodebuild_destination is None: continue
+
+                if target_platform in ["iOS", "visionOS"]:
+                    exclusivity_keys.append(target_platform)
+
+                all_combinations.append({"target_platform" : target_platform, "config" : config, 
+                                         "build_system" : build_system, "xcodebuild_destination" : xcodebuild_destination,
+                                         "exclusivity_keys" : exclusivity_keys})
 
     random.shuffle(all_combinations)
 

.github/scripts/release-new-version.py

@@ -0,0 +1,90 @@
+#===----------------------------------------------------------------------===#
+# This source file is part of github.com/apple/SwiftUsd
+#
+# Copyright © 2025 Apple Inc. and the SwiftUsd project authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+#===----------------------------------------------------------------------===#
+
+from swiftusd_ci_common import *
+import argparse
+import re
+
+def quiet_run(args):
+    return run(args, cwd=Environment.Path.swiftusd, logOutput=False)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("swiftusd_tag")
+    args = parser.parse_args()
+
+    unstaged =  quiet_run(["git", "status", "--porcelain=v1"]).output
+    modified_files = []
+    for l in unstaged:
+        # porcelain format: <xy> <path> or <xy> <orig-path> -> <path>,
+        # where xy is a two-character status code
+        l = l[3:]
+        if m := re.match("(.*)->(.*)", l):
+            modified_files.append(m.group(1).strip())
+            modified_files.append(m.group(2).strip())
+        else:
+            modified_files.append(l.strip())
+
+    modified_roots = set()
+    for f in modified_files:
+        if m := re.match("([^/]*)/.*", f):
+            modified_roots.add(m.group(1))
+        else:
+            modified_roots.add(f)
+
+    required_roots = ["swift-package", "docs", "SwiftUsd.doccarchive"]
+    expected_roots = ["Package.swift"]
+    had_error = False
+
+    for r in required_roots:
+        if r not in modified_roots:
+            print(f"Error: {r} was not modified")
+            had_error = True
+        modified_roots.discard(r)
+
+    for r in expected_roots:
+        if r not in modified_roots:
+            print(f"Warning: {r} was not modified")
+        modified_roots.discard(r)
+
+    if len(modified_roots) != 0:
+        print(f"Error: unexpected modifications: {modified_roots}")
+        had_error = True
+
+    if had_error:
+        exit(1)
+
+    quiet_run(["git", "add", "-A"])
+    quiet_run(["git", "commit", "-m", f"Publish documentation and update swift-package for SwiftUsd {args.swiftusd_tag}"])
+    quiet_run(["git", "tag", args.swiftusd_tag])
+
+
+    printAndWrite(summary=f"""To release SwiftUsd {args.swiftusd_tag}, run:
+```
+cd {Environment.Path.swiftusd}
+git push
+git push origin {args.swiftusd_tag}
+```
+Remember to run these commands in SwiftUsd-Tests and SwiftUsd-ast-answerer:
+```
+git tag {args.swiftusd_tag}
+git push
+git push origin {args.swiftusd_tag}
+```""")

.github/scripts/run-tests-helper.py

@@ -45,7 +45,7 @@ def should_extract(l):
         if action == "build":
             return any([s in l for s in ["error:"]])
         elif action == "test":
-            return any([s in l for s in ["failed:", "launchd"]])
+            return any([s in l for s in ["failed:", "launchd", "crash"]])
 
     def processRunResult(runResult):
         end = time.time()
@@ -78,22 +78,52 @@ def processRunResult(runResult):
 
 def prepare_to_build_tests():
     print("Preparing to build tests...")
+    run(["swift", "package", "--package-path", "ReconfigurePbxprojPackageDependency", "clean"],
+        cwd=Environment.Path.swiftusd_tests)
     run(["swift", "run", "--package-path", "ReconfigurePbxprojPackageDependency",
         "ReconfigurePbxprojPackageDependency", "SwiftUsdTests.xcodeproj/project.pbxproj",
          "--replace", "https://github.com/apple/SwiftUsd", "--with", "SwiftUsd"], 
         cwd=Environment.Path.swiftusd_tests)
-    if not (Environment.Path.swiftusd_tests / "SwiftUsd").exists():
-        (Environment.Path.swiftusd_tests / "SwiftUsd").symlink_to(Environment.Path.swiftusd)
+
+    if (Environment.Path.swiftusd_tests / "SwiftUsd").exists():
+        os.unlink(Environment.Path.swiftusd_tests / "SwiftUsd")
+    (Environment.Path.swiftusd_tests / "SwiftUsd").symlink_to(Environment.Path.swiftusd)
     run(["python3", "make-spm-tests.py", "--local", "--force"],
         cwd=Environment.Path.swiftusd_tests)
 
-def get_xcodebuild_destination():
-    if Environment.TestCombination.target_platform == "macOS": return "platform=macOS,name=My Mac"
-    elif Environment.TestCombination.target_platform == "iOSSimulator": return "platform=iOS Simulator,name=iPhone 17 Pro"
-    elif Environment.TestCombination.target_platform == "visionOSSimulator": return "platform=visionOS Simulator,name=Apple Vision Pro (at 2732x2048)"
+def conditionalCommonArgs():
+    result = []
+
+    def handleJobs(x, singleMinus):
+        nonlocal result
+        try:
+            if int(x) > 0:
+                result += ["-jobs" if singleMinus else "--jobs", x]
+        except:
+            pass        
+
+    def handleDevelopmentTeam():
+        nonlocal result
+        if Environment.TestCombination.at_desk_development_team:
+            result += [f"DEVELOPMENT_TEAM={Environment.TestCombination.at_desk_development_team}"]
+
+    if Environment.TestCombination.build_system == "xcodebuild-xcodeproj":
+        handleJobs(Environment.TestCombination.at_desk_xcodebuild_jobs, True)
+        handleDevelopmentTeam()
+        
+    elif Environment.TestCombination.build_system == "swiftbuild-SPM-Tests":
+        handleJobs(Environment.TestCombination.at_desk_swiftbuild_jobs, False)
+        
+    elif Environment.TestCombination.build_system == "xcodebuild-SPM-Tests":
+        handleJobs(Environment.TestCombination.at_desk_xcodebuild_jobs, True)
+        handleDevelopmentTeam()
+        
     else:
-        print(f"Error: Unknown target '{Environment.TestCombination.target_platform}'")
+        print(f"Error: Unknown build system {Environment.TestCombination.build_system}")
         exit(1)
+    
+    
+    return result
 
 def do_xcodebuild_xcodeproj_tests(action):
     build_or_run_test_suite(
@@ -106,11 +136,11 @@ def do_xcodebuild_xcodeproj_tests(action):
         ],
         buildTestCommonArgs=[
             "-verbose", "-skipMacroValidation",
-            "-scheme", "UnitTests", 
+            "-scheme", "UnitTests",
             "-configuration", Environment.TestCombination.config,
-            "-destination", get_xcodebuild_destination(),
+            "-destination", Environment.TestCombination.xcodebuild_destination,
             "OTHER_SWIFT_FLAGS=$(inherited) -DSWIFTUSD_TESTS_SUPPRESS_PERFORMANCE_FAILURES",
-        ],
+        ] + conditionalCommonArgs(),
         cwd=Environment.Path.swiftusd_tests,
         action=action,
     )
@@ -119,10 +149,14 @@ def do_swiftbuild_spm_tests(action):
     env = os.environ.copy()
     env["SWIFT_BACKTRACE"] = "interactive=no"
 
-    # Set DEVELOPER_DIR to work around 
-    # error: cannot load module 'SwiftCompilerPlugin' built with SDK 'macosx26.0' when using SDK 'macosx26.2'
-    # https://github.com/apple/SwiftUsd/actions/runs/21256906902/job/61175337955
-    env["DEVELOPER_DIR"] = "/Applications/Xcode-latest.app"
+    # todo: revisit this, setting it here means that
+    # xcodebuild -version in the logs is misleading because
+    # it's computed without this environment variable
+    if os.path.exists("/Applications/Xcode-latest.app"):
+        # Set DEVELOPER_DIR to work around 
+        # error: cannot load module 'SwiftCompilerPlugin' built with SDK 'macosx26.0' when using SDK 'macosx26.2'
+        # https://github.com/apple/SwiftUsd/actions/runs/21256906902/job/61175337955
+        env["DEVELOPER_DIR"] = "/Applications/Xcode-latest.app"
 
     build_or_run_test_suite(
         name="swiftbuild-SPM-Tests",
@@ -134,7 +168,7 @@ def do_swiftbuild_spm_tests(action):
             "-Xswiftc", "-DOPENUSD_SWIFT_BUILD_FROM_CLI", "-Xcxx", "-DOPENUSD_SWIFT_BUILD_FROM_CLI",
             "--configuration", Environment.TestCombination.config.lower(),
             "-Xswiftc", "-DSWIFTUSD_TESTS_SUPPRESS_PERFORMANCE_FAILURES",
-        ],
+        ] + conditionalCommonArgs(),
         cwd=Environment.Path.swiftusd_tests / "SPM-Tests",
         action=action,
         env=env
@@ -153,9 +187,9 @@ def do_xcodebuild_spm_tests(action):
             "-verbose", "-skipMacroValidation",
             "-scheme", "SPM-Tests-Package",
             "-config", Environment.TestCombination.config,
-            "-destination", get_xcodebuild_destination(),
+            "-destination", Environment.TestCombination.xcodebuild_destination,
             "OTHER_SWIFT_FLAGS=$(inherited) -DSWIFTUSD_TESTS_SUPPRESS_PERFORMANCE_FAILURES",
-        ],
+        ] + conditionalCommonArgs(),
         cwd=Environment.Path.swiftusd_tests / "SPM-Tests",
         action=action,
     )
@@ -198,4 +232,4 @@ def run_tests(args, subparsers):
 
     args = parser.parse_args()
 
-    args.func(args)
+    args.func(args)

.github/scripts/swiftusd_ci_common/environment.py

@@ -37,6 +37,12 @@ class TestCombination:
         config = os.getenv("CONFIG")
         build_system = os.getenv("BUILD_SYSTEM")
         github_run_id = os.getenv("GITHUB_RUN_ID")
+        xcodebuild_destination = os.getenv("XCODEBUILD_DESTINATION")
+        at_desk_iOS_xcodebuild_destination = os.getenv("ATDESK_IOS_XCODEBUILD_DESTINATION")
+        at_desk_visionOS_xcodebuild_destination = os.getenv("ATDESK_VISIONOS_XCODEBUILD_DESTINATION")
+        at_desk_swiftbuild_jobs = os.getenv("ATDESK_SWIFTBUILD_JOBS")
+        at_desk_xcodebuild_jobs = os.getenv("ATDESK_XCODEBUILD_JOBS")
+        at_desk_development_team = os.getenv("ATDESK_DEVELOPMENT_TEAM")
 
     class Path:
         swiftusd = _getenvpath("SWIFTUSD_PATH")
@@ -48,4 +54,4 @@ class Path:
         tmp_dir = _getenvpath("RUNNER_TEMP")
         github_workspace = _getenvpath("GITHUB_WORKSPACE")
         matrix_result = _getenvpath("MATRIX_RESULT_PATH")
-        matrix_results = _getenvpath("MATRIX_RESULTS_PATH")
+        matrix_results = _getenvpath("MATRIX_RESULTS_PATH")

.github/scripts/swiftusd_ci_common/openusd_building.py

@@ -32,24 +32,26 @@ def clone_openusd(checkout=None):
         run(["git", "checkout", checkout], cwd=Environment.Path.openusd, logOutput=False)
 
 def get_openusd_build_flags(target):
+    file_prefix_map = f"--build-args=USD,\"-DCMAKE_CXX_FLAGS_INIT=-ffile-prefix-map={Environment.Path.openusd}=OpenUSD\""
+    
     if target == "macOS":
         return ["--embree", "--imageio", "--alembic", "--openvdb", "--no-python",
-                "--ignore-homebrew", "--build-target", "native", openusd_build_dir("macOS")]
+                "--ignore-homebrew", "--build-target", "native", openusd_build_dir("macOS"), file_prefix_map]
 
     if target == "iOS":
         return ["--imageio", "--alembic", "--no-python", "--ignore-homebrew",
-                "--build-target", "iOS", openusd_build_dir("iOS")]
+                "--build-target", "iOS", openusd_build_dir("iOS"), file_prefix_map]
 
     if target == "iOSSimulator":
         return ["--imageio", "--alembic", "--no-python", "--ignore-homebrew",
-                "--build-target", "iOSSimulator", openusd_build_dir("iOSSimulator")]
+                "--build-target", "iOSSimulator", openusd_build_dir("iOSSimulator"), file_prefix_map]
 
     if target == "visionOS":
         return ["--imageio", "--alembic", "--no-python", "--ignore-homebrew",
-                "--build-target", "visionOS", openusd_build_dir("visionOS")]
+                "--build-target", "visionOS", openusd_build_dir("visionOS"), file_prefix_map]
 
     if target == "visionOSSimulator":
         return ["--imageio", "--alembic", "--no-python", "--ignore-homebrew",
-                "--build-target", "visionOSSimulator", openusd_build_dir("visionOSSimulator")]
+                "--build-target", "visionOSSimulator", openusd_build_dir("visionOSSimulator"), file_prefix_map]
 
-    raise ValueError(f"Unknown target {target}")
+    raise ValueError(f"Unknown target {target}")

.github/scripts/swiftusd_ci_common/subprocesses.py

@@ -67,4 +67,7 @@ def run(args, cwd=None, env=None, input=None, logCmd=True, logOutput=True, check
 
         exit(result.returncode)
 
-    return result
+    return result
+
+def which(cmd):
+    return run(["which", cmd], logCmd=False, logOutput=False, check=False).returncode == 0

.github/scripts/swiftusd_ci_common/test_matrix_results.py

@@ -40,12 +40,12 @@ def __init__(self):
         raw_xcodebuild_version = "\n".join(run(["xcodebuild", "-version"]).output)
         short_xcodebuild_version = re.search(r"Xcode (.*)", raw_xcodebuild_version).group(1) + " (" + re.search(r"Build version (.*)", raw_xcodebuild_version).group(1) + ")"
         if Environment.GitRef.swiftusd and Environment.Path.swiftusd:
-            swiftusd_ref = run(["git", "rev-parse", Environment.GitRef.swiftusd], cwd=Environment.Path.swiftusd).output[0]
+            swiftusd_ref = run(["git", "rev-parse", Environment.GitRef.swiftusd], cwd=Environment.Path.swiftusd, check=False).output[0]
         else:
             swiftusd_ref = "null"
 
         if Environment.GitRef.swiftusd_tests and Environment.Path.swiftusd_tests:
-            swiftusd_tests_ref = run(["git", "rev-parse", Environment.GitRef.swiftusd_tests], cwd=Environment.Path.swiftusd_tests).output[0]
+            swiftusd_tests_ref = run(["git", "rev-parse", Environment.GitRef.swiftusd_tests], cwd=Environment.Path.swiftusd_tests, check=False).output[0]
         else:
             swiftusd_tests_ref = "null"