Merge branch 'main' into fix/matmul4bit-gemv-shape-guard

Author: datavorous

.github/scripts/build-rocm.sh

@@ -21,6 +21,33 @@ if [ "${build_os:0:6}" == ubuntu ]; then
       && pip install cmake==3.31.6 \
       && cmake -DCOMPUTE_BACKEND=hip -DCMAKE_BUILD_TYPE=MinSizeRel -DCMAKE_HIP_FLAGS=\"--offload-compress\" -DBNB_ROCM_ARCH=\"${bnb_rocm_arch}\" . \
       && cmake --build ."
+else
+    bnb_rocm_arch="gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1200;gfx1201"
+
+    pip install ninja cmake==3.31.6
+
+    # Install ROCm SDK wheels from repo.radeon.com.
+    rocm_base_url="https://repo.radeon.com/rocm/windows/rocm-rel-${rocm_version}"
+    pip install \
+        "${rocm_base_url}/rocm_sdk_core-${rocm_version}-py3-none-win_amd64.whl" \
+        "${rocm_base_url}/rocm_sdk_devel-${rocm_version}-py3-none-win_amd64.whl" \
+        "${rocm_base_url}/rocm_sdk_libraries_custom-${rocm_version}-py3-none-win_amd64.whl" \
+        "${rocm_base_url}/rocm-${rocm_version}.tar.gz"
+
+    # Expand the devel tarball
+    rocm-sdk init
+
+    ROCM_PATH="$(rocm-sdk path --root)"
+    export ROCM_PATH
+    export PATH="${ROCM_PATH}/bin:${PATH}"
+
+    cmake -G Ninja \
+        -DCOMPUTE_BACKEND=hip \
+        -DBNB_ROCM_ARCH="${bnb_rocm_arch}" \
+        -DCMAKE_BUILD_TYPE=MinSizeRel \
+        -DCMAKE_HIP_FLAGS="--offload-compress" \
+        -S .
+    cmake --build .
 fi
 
 output_dir="output/${build_os}/${build_arch}"

.github/workflows/python-package.yml

@@ -136,11 +136,16 @@ jobs:
       matrix:
         os: [ubuntu-22.04]
         arch: [x86_64]
-        rocm_version: ["6.2.4", "6.3.4", "6.4.4", "7.0.2", "7.1", "7.2"]
+        rocm_version: ["6.2.4", "6.3.4", "6.4.4", "7.0.2", "7.1", "7.2.1"]
+        include:
+          - os: windows-2025
+            arch: x86_64
+            rocm_version: "7.2.1"
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
       - name: Clean up disk space
+        if: startsWith(matrix.os, 'ubuntu')
         run: |
           echo "Disk space before cleanup:"
           df -h
@@ -156,6 +161,9 @@ jobs:
 
           echo "Disk space after cleanup:"
           df -h
+      - name: Setup MSVC
+        if: startsWith(matrix.os, 'windows')
+        uses: ilammy/msvc-dev-cmd@v1.13.0
       - name: Build C++
         run: bash .github/scripts/build-rocm.sh
         env:
@@ -332,6 +340,12 @@ jobs:
           done
 
           cat >> body.md << 'ENDOFMARKDOWN'
+          > **Custom PyTorch builds (Intel XPU, ROCm, etc.):**
+          > The `--force-reinstall` flag causes pip to re-resolve all dependencies from PyPI, which may replace your custom PyTorch build with the default CUDA variant. To avoid this, add `--no-deps`:
+          > ```sh
+          > pip install --force-reinstall --no-deps https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl
+          > ```
+
           > **Note:**
           > These wheels are updated automatically with every commit to `main` and become available as soon as the [python-package.yml](.github/workflows/python-package.yml) workflow finishes.
 

.github/workflows/tests-nightly.yml

@@ -20,7 +20,7 @@ jobs:
         platform: [linux-x64, linux-aarch64, macos, windows]
         # default runners don't have AVX-512 support, but icelake does
         cpu_type: ["", icelake]
-        torch_version: ["2.3.1", "2.9.1", "2.10.0"]
+        torch_version: ["2.3.1", "2.10.0", "2.11.0"]
 
         exclude:
           # aarch64 minimum torch version is 2.5.1
@@ -71,7 +71,7 @@ jobs:
             torch_version: "2.9.1"
             pypi_index: "https://download.pytorch.org/whl/cu128"
           - cuda_version: "13.0.2"
-            torch_version: "2.10.0"
+            torch_version: "2.11.0"
             pypi_index: "https://download.pytorch.org/whl/cu130"
 
           # Windows CUDA Tests - T4 GPU (CUDA 11.8 only, multiple torch versions)

.github/workflows/tests-pr.yml

@@ -31,7 +31,7 @@ jobs:
         platform: [linux-x64, linux-aarch64, macos]
         # default runners don't have AVX-512 support, but icelake does
         cpu_type: ["", icelake]
-        torch_version: ["2.3.1", "2.10.0"]
+        torch_version: ["2.3.1", "2.11.0"]
 
         exclude:
           # aarch64 minimum torch version is 2.5.1
@@ -76,7 +76,7 @@ jobs:
             torch_version: "2.9.1"
             pypi_index: "https://download.pytorch.org/whl/cu128"
           - cuda_version: "13.0.2"
-            torch_version: "2.10.0"
+            torch_version: "2.11.0"
             pypi_index: "https://download.pytorch.org/whl/cu130"
 
           # Windows CUDA test - single configuration

CMakeLists.txt

@@ -19,14 +19,17 @@ cmake_minimum_required(VERSION 3.22.1)
 # On Windows with HIP backend, auto-detect compilers from ROCM_PATH before project()
 if(WIN32 AND COMPUTE_BACKEND STREQUAL "hip")
     if(DEFINED ENV{ROCM_PATH})
-        set(ROCM_PATH $ENV{ROCM_PATH})
+        file(TO_CMAKE_PATH "$ENV{ROCM_PATH}" ROCM_PATH)
     endif()
     if(ROCM_PATH AND NOT DEFINED CMAKE_CXX_COMPILER)
         set(CMAKE_CXX_COMPILER "${ROCM_PATH}/lib/llvm/bin/clang++.exe")
     endif()
     if(ROCM_PATH AND NOT DEFINED CMAKE_HIP_COMPILER)
         set(CMAKE_HIP_COMPILER "${ROCM_PATH}/lib/llvm/bin/clang++.exe")
     endif()
+    if(NOT DEFINED HIP_PLATFORM)
+        set(HIP_PLATFORM "amd" CACHE STRING "HIP Platform")
+    endif()
     # On Windows, the HIP compiler needs explicit paths to find device libraries.
     if(ROCM_PATH)
         find_path(ROCM_DEVICE_LIB_PATH
@@ -35,9 +38,9 @@ if(WIN32 AND COMPUTE_BACKEND STREQUAL "hip")
                   "${ROCM_PATH}/lib/llvm/amdgcn/bitcode"
             NO_DEFAULT_PATH
         )
-        set(CMAKE_HIP_FLAGS "--rocm-path=${ROCM_PATH}")
+        string(APPEND CMAKE_HIP_FLAGS " --rocm-path=${ROCM_PATH}")
         if(ROCM_DEVICE_LIB_PATH)
-            set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} --rocm-device-lib-path=${ROCM_DEVICE_LIB_PATH}")
+            string(APPEND CMAKE_HIP_FLAGS " --rocm-device-lib-path=${ROCM_DEVICE_LIB_PATH}")
         endif()
     endif()
 endif()
@@ -357,7 +360,7 @@ endif()
 if(BUILD_HIP)
     # Determine ROCM_PATH from environment variable, fallback to /opt/rocm on Linux
     if(DEFINED ENV{ROCM_PATH})
-      set(ROCM_PATH $ENV{ROCM_PATH})
+      file(TO_CMAKE_PATH "$ENV{ROCM_PATH}" ROCM_PATH)
     else()
       set(ROCM_PATH /opt/rocm)
     endif()
@@ -416,11 +419,15 @@ if(WIN32)
     set_target_properties(bitsandbytes PROPERTIES PREFIX "lib")
 endif()
 set_target_properties(bitsandbytes PROPERTIES OUTPUT_NAME ${BNB_OUTPUT_NAME})
-if(MSVC)
-    set_target_properties(bitsandbytes PROPERTIES LIBRARY_OUTPUT_DIRECTORY_RELEASE "${PROJECT_SOURCE_DIR}/bitsandbytes")
-    set_target_properties(bitsandbytes PROPERTIES LIBRARY_OUTPUT_DIRECTORY_DEBUG "${PROJECT_SOURCE_DIR}/bitsandbytes")
-    set_target_properties(bitsandbytes PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE "${PROJECT_SOURCE_DIR}/bitsandbytes")
-    set_target_properties(bitsandbytes PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG "${PROJECT_SOURCE_DIR}/bitsandbytes")
+if(WIN32)
+    set_target_properties(bitsandbytes PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${PROJECT_SOURCE_DIR}/bitsandbytes")
+    set_target_properties(bitsandbytes PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${PROJECT_SOURCE_DIR}/bitsandbytes")
+    if(MSVC)
+        set_target_properties(bitsandbytes PROPERTIES LIBRARY_OUTPUT_DIRECTORY_RELEASE "${PROJECT_SOURCE_DIR}/bitsandbytes")
+        set_target_properties(bitsandbytes PROPERTIES LIBRARY_OUTPUT_DIRECTORY_DEBUG "${PROJECT_SOURCE_DIR}/bitsandbytes")
+        set_target_properties(bitsandbytes PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE "${PROJECT_SOURCE_DIR}/bitsandbytes")
+        set_target_properties(bitsandbytes PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG "${PROJECT_SOURCE_DIR}/bitsandbytes")
+    endif()
 endif()
 
 set_target_properties(bitsandbytes PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${PROJECT_SOURCE_DIR}/bitsandbytes")

README.md

@@ -57,7 +57,7 @@ bitsandbytes has the following minimum requirements for all platforms:
       <td>Minimum: AVX2<br>Optimized: AVX512F, AVX512BF16</td>
       <td>✅</td>
       <td>✅</td>
-      <td>❌</td>
+      <td>✅</td>
     </tr>
     <tr>
       <td></td>
@@ -88,7 +88,7 @@ bitsandbytes has the following minimum requirements for all platforms:
       </td>
       <td>✅</td>
       <td>✅</td>
-      <td>〰️</td>
+      <td>✅</td>
     </tr>
     <tr>
       <td></td>
@@ -123,7 +123,7 @@ bitsandbytes has the following minimum requirements for all platforms:
       <td>AVX2</td>
       <td>✅</td>
       <td>✅</td>
-      <td>❌</td>
+      <td>✅</td>
     </tr>
     <tr>
       <td></td>
@@ -133,6 +133,18 @@ bitsandbytes has the following minimum requirements for all platforms:
       <td>✅</td>
       <td>✅</td>
     </tr>
+    <tr>
+      <td></td>
+      <td>🟥 AMD GPU <br><code>cuda</code></td>
+      <td>
+        RDNA: gfx1100, gfx1101, gfx1102,<br>
+        gfx1150, gfx1151,<br>
+        gfx1200, gfx1201
+      </td>
+      <td>✅</td>
+      <td>✅</td>
+      <td>✅</td>
+    </tr>
     <tr>
       <td></td>
       <td>🟦 Intel GPU <br><code>xpu</code></td>
@@ -142,7 +154,7 @@ bitsandbytes has the following minimum requirements for all platforms:
       </td>
       <td>✅</td>
       <td>✅</td>
-      <td>〰️</td>
+      <td>✅</td>
     </tr>
     <tr>
       <td colspan="6">🍎 <strong>macOS 14+</strong></td>

bitsandbytes/autograd/_functions.py

@@ -382,10 +382,7 @@ def matmul_4bit(
     bias: Optional[torch.Tensor] = None,
 ):
     assert quant_state is not None
-    # Change dtype to input dtype on CPU
     if A.device.type == "cpu":
-        quant_state.dtype = A.dtype
-
         if getattr(quant_state, "packing_format_for_cpu", False):
             out = F.gemv_4bit(A, B, out, state=quant_state)
             if bias is not None: