build-wheels-cuda.yaml

name: Build Wheels (CUDA)

on: workflow_dispatch

permissions:
  contents: write

jobs:
  define_matrix:
    name: Define Build Matrix
    runs-on: ubuntu-22.04
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    defaults:
      run:
        shell: pwsh

    steps:
      - name: Define Job Output
        id: set-matrix
        run: |
          $matrix = @{
              'os' = @('ubuntu-22.04', 'windows-2022')
              # wheel.py-api = "py3" makes the CUDA wheel interpreter-agnostic,
              # so one builder per toolkit version is sufficient.
              'pyver' = @("3.9")
              'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.1")
              'releasetag' = @("basic")
          }

          $matrixOut = ConvertTo-Json $matrix -Compress
          Write-Output ('matrix=' + $matrixOut) >> $env:GITHUB_OUTPUT

  build_wheels:
    name: Build Wheel ${{ matrix.os }} ${{ matrix.pyver }} ${{ matrix.cuda }} ${{ matrix.releasetag == 'wheels' && 'AVX2' || matrix.releasetag }}
    needs: define_matrix
    runs-on: ${{ matrix.os }}
    strategy:
      matrix: ${{ fromJSON(needs.define_matrix.outputs.matrix) }}
    defaults:
      run:
        shell: pwsh
    env:
      CUDAVER: ${{ matrix.cuda }}
      AVXVER: ${{ matrix.releasetag }}

    steps:
      - name: Set up MSVC
        if: runner.os == 'Windows'
        uses: ilammy/msvc-dev-cmd@v1
        with:
          arch: x64

      - uses: actions/checkout@v4
        with:
          submodules: "recursive"

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.pyver }}
          cache: 'pip'

      - name: Setup Mamba
        uses: conda-incubator/setup-miniconda@v3.1.0
        with:
          activate-environment: "llamacpp"
          python-version: ${{ matrix.pyver }}
          miniforge-version: latest
          add-pip-as-python-dependency: true
          auto-activate-base: false

      - name: Install Dependencies
        env:
          MAMBA_DOWNLOAD_FAILFAST: "0"
          MAMBA_NO_LOW_SPEED_LIMIT: "1"
        run: |
          $cudaVersion = $env:CUDAVER
          $cudaChannel = "nvidia/label/cuda-$cudaVersion"
          if ($IsLinux) {
            mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-toolkit=$cudaVersion" "${cudaChannel}::cuda-nvcc_linux-64" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev"
          } elseif ($IsWindows) {
            if ($cudaVersion -like '12.5.*') {
              # The Windows 12.5 toolkit meta-package pulls compiler activation
              # scripts that overflow cmd.exe after MSVC is already initialized.
              mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-nvcc_win-64" "${cudaChannel}::cuda-libraries-dev=$cudaVersion" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev"
            } else {
              mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-toolkit=$cudaVersion" "${cudaChannel}::cuda-nvcc_win-64" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev"
            }
          } else {
            throw 'Unsupported CUDA wheel build platform'
          }
          if ($LASTEXITCODE -ne 0) {
            exit $LASTEXITCODE
          }
          if ($IsWindows) {
            python -m pip install build wheel ninja
          } else {
            python -m pip install build wheel
          }

      - name: Build Wheel
        run: |
          $pathSeparator = if ($IsWindows) { ';' } else { ':' }
          if ($IsWindows) {
            $cudaRoot = Join-Path $env:CONDA_PREFIX 'Library'
          } elseif (Test-Path (Join-Path $env:CONDA_PREFIX 'targets/x86_64-linux/include/cuda_runtime.h')) {
            $cudaRoot = Join-Path $env:CONDA_PREFIX 'targets/x86_64-linux'
          } else {
            $cudaRoot = $env:CONDA_PREFIX
          }

          $env:CUDA_PATH = $cudaRoot
          $env:CUDA_HOME = $cudaRoot
          $env:CUDAToolkit_ROOT = $cudaRoot
          $env:CUDA_TOOLKIT_ROOT_DIR = $cudaRoot
          $cudaHostCompilerArg = ''
          $cudaRootCmake = $cudaRoot.Replace('\', '/')
          $env:CMAKE_ARGS = "-DCUDAToolkit_ROOT=$cudaRootCmake -DCUDA_TOOLKIT_ROOT_DIR=$cudaRootCmake"
          if ($IsLinux) {
            if (Test-Path '/usr/bin/g++-12') {
              $env:CC = '/usr/bin/gcc-12'
              $env:CXX = '/usr/bin/g++-12'
              $env:CUDAHOSTCXX = '/usr/bin/g++-12'
              $cudaHostCompilerArg = " -DCMAKE_CUDA_HOST_COMPILER=$env:CUDAHOSTCXX"
            }
            $env:CMAKE_ARGS = "-DCUDAToolkit_ROOT=$cudaRoot -DCUDA_TOOLKIT_ROOT_DIR=$cudaRoot$cudaHostCompilerArg"
            $env:CPATH = "$cudaRoot/include$pathSeparator$env:CPATH"
            $env:CPLUS_INCLUDE_PATH = "$cudaRoot/include$pathSeparator$env:CPLUS_INCLUDE_PATH"
            $env:LIBRARY_PATH = "$cudaRoot/lib$pathSeparator$env:CONDA_PREFIX/lib$pathSeparator$env:LIBRARY_PATH"
            $env:LD_LIBRARY_PATH = "$cudaRoot/lib$pathSeparator$env:CONDA_PREFIX/lib$pathSeparator$env:LD_LIBRARY_PATH"
          } elseif ($IsWindows) {
            $ninjaPath = ((Get-Command ninja -ErrorAction Stop).Source).Replace('\', '/')
            $env:CMAKE_GENERATOR = 'Ninja'
            $env:CMAKE_MAKE_PROGRAM = $ninjaPath
            $env:PATH = "$(Join-Path $cudaRoot 'bin')$pathSeparator$env:PATH"
          }

          if ($IsWindows) {
            $nvccCandidates = @(
              (Join-Path $cudaRoot 'bin\nvcc.exe'),
              (Join-Path $env:CONDA_PREFIX 'Library\bin\nvcc.exe'),
              (Join-Path $env:CONDA_PREFIX 'bin\nvcc.exe')
            )
          } else {
            $nvccCandidates = @(
              (Join-Path $env:CONDA_PREFIX 'bin/nvcc'),
              (Join-Path $env:CONDA_PREFIX 'targets/x86_64-linux/bin/nvcc')
            )
          }
          $nvccPath = $nvccCandidates | Where-Object { Test-Path $_ } | Select-Object -First 1
          if (-not $nvccPath) {
            throw 'Failed to find nvcc in the conda environment'
          }
          $env:CUDACXX = $nvccPath
          $env:PATH = "$(Split-Path $nvccPath)$pathSeparator$env:PATH"
          if ($IsWindows) {
            $nvccPathCmake = $nvccPath.Replace('\', '/')
            $env:CUDACXX = $nvccPathCmake
            $env:CMAKE_ARGS = "-DCMAKE_CUDA_COMPILER=$nvccPathCmake -DCMAKE_MAKE_PROGRAM=$env:CMAKE_MAKE_PROGRAM $env:CMAKE_ARGS"
          }
          $nvccVersion = ((& $nvccPath --version) | Select-String 'release ([0-9]+\.[0-9]+)').Matches[0].Groups[1].Value
          if (-not $nvccVersion) {
            throw 'Failed to detect the installed CUDA toolkit version'
          }
          $cudaTagVersion = $nvccVersion.Replace('.','')
          $env:VERBOSE = '1'
          # Build real cubins for the supported GPUs, including sm_70, and keep
          # one forward-compatible PTX target instead of embedding PTX for every
          # SM. This keeps the wheel under GitHub's 2 GiB release-asset limit.
          $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=70-real;75-real;80-real;86-real;89-real;90-real;90-virtual -DCMAKE_CUDA_FLAGS=--allow-unsupported-compiler $env:CMAKE_ARGS"
          $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off'
          python -m build --wheel
          # Publish tags that reflect the actual installed toolkit version.
          Write-Output "CUDA_VERSION=$cudaTagVersion" >> $env:GITHUB_ENV

      - uses: softprops/action-gh-release@v2
        if: startsWith(github.ref, 'refs/tags/')
        with:
          files: dist/*
          # Set tag_name to <tag>-cu<cuda_version>
          tag_name: ${{ github.ref_name }}-cu${{ env.CUDA_VERSION }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}