pep 517 wheels

Author: tylerm-nv

.gitignore

@@ -1,4 +1,7 @@
 .p4*
 .DS_Store
 .AppleDouble
-
+.wheel-build
+.venv_openusd_packaging
+dist
+packaging/**/*.pyc

build_scripts/build_usd.py

@@ -223,21 +223,28 @@ def GetPythonInfo(context):
     pythonVersion = sysconfig.get_config_var("py_version_short")  # "3.7"
 
     # Lib path is unfortunately special for each platform and there is no
-    # config_var for it. But we can deduce it for each platform, and this
-    # logic works for any Python version.
-    def _GetPythonLibraryFilename(context):
+    # single config_var that works everywhere. Build an ordered list of
+    # candidate filenames and then pick the first one that actually exists.
+    def _GetPythonLibraryFilenames(context):
         if Windows():
-            return "python{version}{suffix}.lib".format(
+            return ["python{version}{suffix}.lib".format(
                 version=sysconfig.get_config_var("py_version_nodot"),
                 suffix=('_d' if context.buildDebug and context.debugPython
                         else ''))
+            ]
         elif Linux():
-            return sysconfig.get_config_var("LDLIBRARY")
+            candidates = [
+                sysconfig.get_config_var("LDLIBRARY"),
+                sysconfig.get_config_var("INSTSONAME"),
+                sysconfig.get_config_var("LIBRARY"),
+            ]
+            return [c for i, c in enumerate(candidates) if c and c not in candidates[:i]]
         elif MacOS():
-            return "libpython{version}.dylib".format(
+            return ["libpython{version}.dylib".format(
                 version=(sysconfig.get_config_var('LDVERSION') or
                          sysconfig.get_config_var('VERSION') or
                          pythonVersion))
+            ]
         else:
             raise RuntimeError("Platform not supported")
 
@@ -250,27 +257,47 @@ def _GetPythonLibraryFilename(context):
     # if in a venv, installed_base will be the "original" python,
     # which is where the libs are ("base" will be the venv dir)
     pythonBaseDir = sysconfig.get_config_var("installed_base")
+    pythonLibPath = None
     if Windows():
         pythonLibPath = os.path.join(pythonBaseDir, "libs",
-                                     _GetPythonLibraryFilename(context))
+                                     _GetPythonLibraryFilenames(context)[0])
     elif Linux():
         pythonMultiarchSubdir = sysconfig.get_config_var("multiarchsubdir")
-        # Try multiple ways to get the python lib dir
-        for pythonLibDir in (sysconfig.get_config_var("LIBDIR"),
-                             os.path.join(pythonBaseDir, "lib")):
-            if pythonMultiarchSubdir:
-                pythonLibPath = \
-                    os.path.join(pythonLibDir + pythonMultiarchSubdir,
-                                 _GetPythonLibraryFilename(context))
-                if os.path.isfile(pythonLibPath):
+        pythonLibDirs = []
+
+        def _AppendPythonLibDir(path):
+            if path and path not in pythonLibDirs:
+                pythonLibDirs.append(path)
+
+        def _AppendMultiarchPythonLibDir(path):
+            if not path or not pythonMultiarchSubdir:
+                return
+
+            multiarchSubdir = pythonMultiarchSubdir.lstrip(os.sep)
+            normalizedPath = os.path.normpath(path)
+            if os.path.basename(normalizedPath) == multiarchSubdir:
+                return
+            _AppendPythonLibDir(os.path.join(path, multiarchSubdir))
+
+        libDir = sysconfig.get_config_var("LIBDIR")
+        baseLibDir = os.path.join(pythonBaseDir, "lib")
+
+        _AppendPythonLibDir(libDir)
+        _AppendMultiarchPythonLibDir(libDir)
+        _AppendPythonLibDir(baseLibDir)
+        _AppendMultiarchPythonLibDir(baseLibDir)
+
+        for pythonLibDir in pythonLibDirs:
+            for pythonLibFilename in _GetPythonLibraryFilenames(context):
+                candidateLibPath = os.path.join(pythonLibDir, pythonLibFilename)
+                if os.path.isfile(candidateLibPath):
+                    pythonLibPath = candidateLibPath
                     break
-            pythonLibPath = os.path.join(pythonLibDir,
-                                         _GetPythonLibraryFilename(context))
-            if os.path.isfile(pythonLibPath):
+            if pythonLibPath:
                 break
     elif MacOS():
         pythonLibPath = os.path.join(pythonBaseDir, "lib",
-                                     _GetPythonLibraryFilename(context))
+                                     _GetPythonLibraryFilenames(context)[0])
     else:
         raise RuntimeError("Platform not supported")
 
@@ -1766,12 +1793,15 @@ def InstallUSD(context, force, buildArgs):
             # itself rather than rely on CMake's heuristics.
             pythonInfo = GetPythonInfo(context)
             if pythonInfo:
-                extraArgs.append('-DPython3_EXECUTABLE="{pyExecPath}"'
-                                 .format(pyExecPath=pythonInfo[0]))
-                extraArgs.append('-DPython3_LIBRARY="{pyLibPath}"'
-                                 .format(pyLibPath=pythonInfo[1]))
-                extraArgs.append('-DPython3_INCLUDE_DIR="{pyIncPath}"'
-                                 .format(pyIncPath=pythonInfo[2]))
+                if pythonInfo[0]:
+                    extraArgs.append('-DPython3_EXECUTABLE="{pyExecPath}"'
+                                     .format(pyExecPath=pythonInfo[0]))
+                if pythonInfo[1]:
+                    extraArgs.append('-DPython3_LIBRARY="{pyLibPath}"'
+                                     .format(pyLibPath=pythonInfo[1]))
+                if pythonInfo[2]:
+                    extraArgs.append('-DPython3_INCLUDE_DIR="{pyIncPath}"'
+                                     .format(pyIncPath=pythonInfo[2]))
         else:
             extraArgs.append('-DPXR_ENABLE_PYTHON_SUPPORT=OFF')
 

packaging/README.md

@@ -0,0 +1,98 @@
+# OpenUSD Pyproject Packaging
+
+This directory contains a `pyproject.toml` based wheel build flow.
+
+The OpenUSD build is created with `build_scripts/build_usd.py`, and a PEP 517 backend stages files from that install tree into a wheel.
+In order to provide robust wheels with proper isolation, ownership of libraries, and dependency links, the wheels are modified.
+This approach was chosen over installing all wheels to a common namespace because of fragility with uninstallation, upgrades, versions, and layout conflicts.
+The cmake build was not extended to support a wheel native build, so the wheels needs to be fixed up by the backend.
+The RPATHs are corrected for the installed layout of the wheels, and platform repair steps still use `auditwheel`, `delocate`, and
+`build_scripts/pypi/updatePluginfos.py`.
+
+The backend is designed to be extensible and support more package definitions.
+
+
+## Packages
+
+The number and content of the packages is easily configurable, the following separation is the current implementation:
+
+- `packaging/usd-core`
+- `packaging/usd-imaging`
+- `packaging/usd-tools`
+- `packaging/usdview`
+
+Each package owns its corresponding shared libraries, bindings, python modules, resources, and metadata.
+
+These packages are staged from an installed OpenUSD tree using explicit manifest globs.
+The file ownership boundaries are intentionally easy to revise during review.
+
+On Linux, native shared-library ownership now follows the package dependency graph during repair:
+
+- `usd-core` repairs as a self-contained base wheel
+- higher-level wheels exclude libraries already exported by their OpenUSD
+  dependencies
+- the backend patches ELF `NEEDED` entries and RPATHs so later wheels resolve
+  back into sibling dependency `*.libs` directories after install
+
+That keeps `usd-imaging`, `usd-tools`, and `usdview` from vendoring duplicate
+copies of the same OpenUSD shared libraries.
+
+For command-line tools, the wheel keeps the real executable under
+`site-packages` and exposes the usual command name through a generated Python
+launcher. That avoids relying on `pip` to preserve executable-to-library
+relative paths when it installs scripts into the environment's `bin/`
+directory.
+
+## Local Build Entry Point
+
+The backend supports two build modes:
+
+- Build OpenUSD directly by invoking `build_scripts/build_usd.py`.
+- Reuse a prebuilt install tree by setting `OPENUSD_PREBUILT_INSTALL_ROOT`.
+
+The second mode is intended for CI integration where the install tree is
+already produced elsewhere in the workflow.
+
+## Helper Script
+
+`packaging/build_wheels.sh` is a convenience entry point for local development.
+It will:
+
+- create a virtual environment,
+- install the packaging prerequisites,
+- build one shared OpenUSD install tree unless `--reuse-install` is set,
+- build one or more wheels into a chosen output directory.
+
+Example usage:
+
+```bash
+packaging/build_wheels.sh
+packaging/build_wheels.sh --packages usd-core,usd-imaging
+packaging/build_wheels.sh --reuse-install --install-root /tmp/openusd-inst
+packaging/build_wheels.sh --build-usd-arg --no-tests
+```
+
+The helper installs the wheel-building tools it needs, and any necessary pip
+packages for OpenUSD. That currently includes `Jinja2` when `usd-tools` is
+requested so the schema-generation tools such as `usdGenSchema` and
+`usdInitSchema` are built, and the Qt-for-Python dependencies needed for
+`usdview`.
+
+On Linux, the repair step uses `auditwheel` and requires `patchelf`. The helper
+installs both into the packaging virtual environment when repair is enabled.
+
+## Alternatives
+
+### Common namespace packages
+
+There are a subset of the OpenUSD libraries on pypi as individual packages (pxr-ar, pxr-sdf, etc.).
+These all install into a common site-packages/pxr/.libs tree, eliminating the need for wheel repair.
+There are downsides, though. Ownership of the files is not managed, uninstallation/upgrade of the packages is brittle,
+and there are collision risks.
+
+Because additional wheel build configuration is needed anyways to support the CLI tools because RPATHs will be broken due to default install layouts,
+this approach was not used here. 
+
+### Single Runtime Package
+
+Similar pypi projects sometimes use a single runtime package with all native code in it, and packages built on top of it are pure python/resources/metadata.

packaging/_backend/openusd_pep517/__init__.py

@@ -0,0 +1,2 @@
+"""Local PEP 517 backend for OpenUSD packaging."""
+

packaging/_backend/openusd_pep517/_templates.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+from string import Template
+
+
+BINARY_LAUNCHER_MODULE = """from __future__ import annotations
+
+import os
+import sys
+import sysconfig
+from pathlib import Path
+
+
+def _library_env_key() -> str:
+    if sys.platform == "win32":
+        return "PATH"
+    if sys.platform == "darwin":
+        return "DYLD_FALLBACK_LIBRARY_PATH"
+    return "LD_LIBRARY_PATH"
+
+
+def _packaged_binary(tool_name: str) -> Path:
+    package_root = Path(__file__).resolve().parent
+    binary_dir = package_root / "bin"
+    if sys.platform == "win32":
+        executable = binary_dir / f"{tool_name}.exe"
+        if executable.exists():
+            return executable
+    return binary_dir / tool_name
+
+
+def _library_search_dirs(platlib: Path, launcher_package: Path) -> list[str]:
+    search_dirs = []
+    if launcher_package.exists():
+        search_dirs.append(str(launcher_package))
+    for libs_dir in sorted(platlib.glob("*.libs")):
+        search_dirs.append(str(libs_dir))
+    return search_dirs
+
+
+def main() -> None:
+    tool_name = Path(sys.argv[0]).name
+    binary = _packaged_binary(tool_name)
+    if not binary.exists():
+        raise SystemExit(f"Could not find packaged OpenUSD tool: {binary}")
+
+    env = os.environ.copy()
+    platlib = Path(sysconfig.get_path("platlib"))
+    search_dirs = _library_search_dirs(platlib, binary.parent)
+    env_key = _library_env_key()
+    if search_dirs:
+        existing = env.get(env_key)
+        if existing:
+            search_dirs.append(existing)
+        env[env_key] = os.pathsep.join(search_dirs)
+
+    os.execvpe(str(binary), [str(binary), *sys.argv[1:]], env)
+"""
+
+
+BUILD_SYSTEM_PYPROJECT = """[build-system]
+requires = ["setuptools>=69", "wheel"]
+build-backend = "setuptools.build_meta"
+"""
+
+
+SETUP_PY_TEMPLATE = Template(
+    """import re
+import sysconfig
+from pathlib import Path
+from setuptools import find_namespace_packages, setup
+from setuptools.command.bdist_wheel import bdist_wheel as _bdist_wheel
+from setuptools.command.install import install as _install
+
+
+class openusd_bdist_wheel(_bdist_wheel):
+    def finalize_options(self):
+        super().finalize_options()
+        self.root_is_pure = False
+
+    def get_tag(self):
+        python_tag, abi_tag, platform_tag = super().get_tag()
+        soabi = sysconfig.get_config_var("SOABI") or ""
+        match = re.match(r"cpython-(\\d+[a-z]*)", soabi)
+        if match:
+            cpython_tag = f"cp{match.group(1)}"
+            return cpython_tag, cpython_tag, platform_tag
+        return python_tag, abi_tag, platform_tag
+
+
+class openusd_install(_install):
+    def finalize_options(self):
+        super().finalize_options()
+        self.install_lib = self.install_platlib
+
+
+root = Path(__file__).resolve().parent
+kwargs = $setup_kwargs_literal
+packages = find_namespace_packages(where="staged/lib/python", include=["pxr*"])
+for helper_package in $helper_packages_literal:
+    if helper_package not in packages:
+        packages.append(helper_package)
+kwargs["packages"] = sorted(packages)
+kwargs["long_description"] = (root / "README.md").read_text(encoding="utf-8")
+kwargs["cmdclass"] = {
+    "bdist_wheel": openusd_bdist_wheel,
+    "install": openusd_install,
+}
+
+data_files = {}
+for path in (root / "staged").rglob("*"):
+    if not path.is_file():
+        continue
+    rel = path.relative_to(root / "staged")
+    if rel.parts[:2] == ("lib", "python"):
+        continue
+    destination = "." if rel.parent == Path(".") else str(rel.parent)
+    data_files.setdefault(destination, []).append(str(path))
+kwargs["data_files"] = sorted((dst, files) for dst, files in data_files.items())
+setup(**kwargs)
+"""
+)