ci: add nightly integration tests for transformers, accelerate, peft #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests (Downstream) | |
| # Nightly smoke tests: run the bnb-specific test suites from transformers, | |
| # accelerate, and peft against the latest main-branch bnb wheel. Catches | |
| # downstream breakage before it reaches users. | |
| # | |
| # bnb is installed from the `continuous-release_main` pre-release which | |
| # python-package.yml publishes on every push to main — no duplicate build. | |
| # | |
| # See agents/integration_tests_guide.md for background. | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| paths: | |
| - '.github/workflows/tests-integration-nightly.yml' | |
| - 'scripts/integration_test_report.py' | |
| # schedule: | |
| # - cron: "30 3 * * *" # enable once stable; runs after python-package + tests-nightly | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| PYTHON_VERSION: "3.10" | |
| TORCH_VERSION: "2.9.1" | |
| PYPI_INDEX: "https://download.pytorch.org/whl/cu128" | |
| BNB_WHEEL_URL: "https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl" | |
| jobs: | |
| # ─── Downstream test jobs ───────────────────────────────────────────────── | |
| # Each job: | |
| # 1. Installs torch, then bnb from the continuous-release wheel | |
| # 2. Installs the downstream lib (latest release from PyPI) | |
| # 3. Clones the matching version tag for the test files | |
| # 4. Runs the library's bnb-specific tests with --junitxml | |
| # 5. Uploads the XML + full log as an artifact for the report job | |
| # | |
| # Runner matching rationale (see integration_tests_guide.md): | |
| # transformers CI runs on T4 → we use T4 | |
| # accelerate / peft CI runs on L4 → closest bnb equivalent is A10 | |
| # This reduces spurious failures from expected values calibrated on their runners. | |
| test-transformers: | |
| name: Transformers bnb tests | |
| if: github.repository == 'bitsandbytes-foundation/bitsandbytes' | |
| runs-on: bandb-aws-g4dn-4xlarge-plus-use1-public-80 # T4 | |
| steps: | |
| - name: Show GPU information | |
| run: nvidia-smi | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install torch + bnb (from continuous-release) | |
| run: | | |
| pip install torch==${TORCH_VERSION} --index-url ${PYPI_INDEX} | |
| pip install "bitsandbytes[test] @ ${BNB_WHEEL_URL}" | |
| - name: Install transformers and clone matching tag | |
| run: | | |
| pip install transformers accelerate | |
| TRANSFORMERS_VERSION=$(pip show transformers | awk '/^Version:/ {print $2}') | |
| echo "Installed transformers v${TRANSFORMERS_VERSION}" | |
| git clone --depth=1 --branch "v${TRANSFORMERS_VERSION}" \ | |
| https://github.com/huggingface/transformers.git /tmp/transformers | |
| - name: Show environment | |
| run: | | |
| pip list | |
| python -m torch.utils.collect_env | |
| - name: Run transformers bnb tests | |
| working-directory: /tmp/transformers | |
| env: | |
| RUN_SLOW: "1" | |
| CUDA_VISIBLE_DEVICES: "0" | |
| shell: bash -o pipefail {0} | |
| run: | | |
| mkdir -p ${GITHUB_WORKSPACE}/reports | |
| python -m pytest tests/quantization/bnb/ \ | |
| -v \ | |
| -k "not MultiGpu and not multi_gpu" \ | |
| --junitxml=${GITHUB_WORKSPACE}/reports/transformers.xml \ | |
| -o junit_logging=all \ | |
| 2>&1 | tee ${GITHUB_WORKSPACE}/reports/transformers.log | |
| - name: Upload JUnit XML and log | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: reports-transformers | |
| path: reports/ | |
| retention-days: 7 | |
| test-accelerate: | |
| name: Accelerate bnb tests | |
| if: github.repository == 'bitsandbytes-foundation/bitsandbytes' | |
| runs-on: bandb-aws-g5-4xlarge-plus-use1-public-80 # A10 | |
| steps: | |
| - name: Show GPU information | |
| run: nvidia-smi | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install torch + bnb (from continuous-release) | |
| run: | | |
| pip install torch==${TORCH_VERSION} --index-url ${PYPI_INDEX} | |
| pip install "bitsandbytes[test] @ ${BNB_WHEEL_URL}" | |
| - name: Install accelerate and clone matching tag | |
| run: | | |
| pip install accelerate transformers | |
| ACCELERATE_VERSION=$(pip show accelerate | awk '/^Version:/ {print $2}') | |
| echo "Installed accelerate v${ACCELERATE_VERSION}" | |
| git clone --depth=1 --branch "v${ACCELERATE_VERSION}" \ | |
| https://github.com/huggingface/accelerate.git /tmp/accelerate | |
| - name: Show environment | |
| run: | | |
| pip list | |
| python -m torch.utils.collect_env | |
| - name: Run accelerate bnb tests | |
| working-directory: /tmp/accelerate | |
| env: | |
| RUN_SLOW: "1" | |
| CUDA_VISIBLE_DEVICES: "0" | |
| shell: bash -o pipefail {0} | |
| run: | | |
| mkdir -p ${GITHUB_WORKSPACE}/reports | |
| python -m pytest tests/test_quantization.py \ | |
| -s -v \ | |
| -k "not multi_device" \ | |
| --junitxml=${GITHUB_WORKSPACE}/reports/accelerate.xml \ | |
| -o junit_logging=all \ | |
| 2>&1 | tee ${GITHUB_WORKSPACE}/reports/accelerate.log | |
| - name: Upload JUnit XML and log | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: reports-accelerate | |
| path: reports/ | |
| retention-days: 7 | |
| test-peft: | |
| name: PEFT bnb tests | |
| if: github.repository == 'bitsandbytes-foundation/bitsandbytes' | |
| runs-on: bandb-aws-g5-4xlarge-plus-use1-public-80 # A10 | |
| steps: | |
| - name: Show GPU information | |
| run: nvidia-smi | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install torch + bnb (from continuous-release) | |
| run: | | |
| pip install torch==${TORCH_VERSION} --index-url ${PYPI_INDEX} | |
| pip install "bitsandbytes[test] @ ${BNB_WHEEL_URL}" | |
| - name: Install peft and clone matching tag | |
| run: | | |
| pip install peft transformers accelerate datasets | |
| PEFT_VERSION=$(pip show peft | awk '/^Version:/ {print $2}') | |
| echo "Installed peft v${PEFT_VERSION}" | |
| git clone --depth=1 --branch "v${PEFT_VERSION}" \ | |
| https://github.com/huggingface/peft.git /tmp/peft | |
| - name: Show environment | |
| run: | | |
| pip list | |
| python -m torch.utils.collect_env | |
| - name: Run peft bnb tests | |
| working-directory: /tmp/peft | |
| env: | |
| IS_GITHUB_CI: "1" | |
| CUDA_VISIBLE_DEVICES: "0" | |
| shell: bash -o pipefail {0} | |
| run: | | |
| mkdir -p ${GITHUB_WORKSPACE}/reports | |
| python -m pytest \ | |
| -m "single_gpu_tests and bitsandbytes" \ | |
| tests/test_gpu_examples.py tests/test_common_gpu.py \ | |
| -v \ | |
| --junitxml=${GITHUB_WORKSPACE}/reports/peft.xml \ | |
| -o junit_logging=all \ | |
| 2>&1 | tee ${GITHUB_WORKSPACE}/reports/peft.log | |
| - name: Upload JUnit XML and log | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: reports-peft | |
| path: reports/ | |
| retention-days: 7 | |
| # ─── Consolidated report ────────────────────────────────────────────────── | |
| # Runs after all three test jobs finish (success or failure). | |
| # Downloads the JUnit XMLs, runs our report script, writes to the job | |
| # summary, and uploads the consolidated report as an artifact. | |
| # Slack posting is intentionally disabled until the SLACK_API_TOKEN secret | |
| # is provisioned — we'll iterate on the Slack payload locally against real | |
| # artifacts first. | |
| report: | |
| name: Consolidated report | |
| needs: [test-transformers, test-accelerate, test-peft] | |
| if: always() && github.repository == 'bitsandbytes-foundation/bitsandbytes' | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Download all report artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts | |
| pattern: reports-* | |
| - name: Consolidate XMLs into reports/ | |
| run: | | |
| mkdir -p reports | |
| # Each artifact lands in artifacts/reports-<suite>/ — flatten to reports/<suite>.xml | |
| find artifacts -name '*.xml' -exec cp {} reports/ \; | |
| find artifacts -name '*.log' -exec cp {} reports/ \; | |
| ls -la reports/ | |
| - name: Generate consolidated report | |
| run: | | |
| python scripts/integration_test_report.py \ | |
| --reports-dir reports/ \ | |
| --output consolidated_report.md \ | |
| --dry-run | |
| - name: Write to job summary | |
| if: always() | |
| run: | | |
| cat consolidated_report.md >> $GITHUB_STEP_SUMMARY | |
| - name: Upload consolidated report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: consolidated-report | |
| path: | | |
| consolidated_report.md | |
| reports/ | |
| retention-days: 14 | |
| # TODO: enable once SLACK_API_TOKEN secret is configured | |
| # - name: Post to Slack | |
| # if: always() | |
| # env: | |
| # SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }} | |
| # run: | | |
| # pip install slack_sdk | |
| # python scripts/integration_test_report.py \ | |
| # --reports-dir reports/ \ | |
| # --slack-channel bnb-ci-nightly |