ci-python.yml

# This workflow runs generic Python tests global to Dagster and PostHog.
name: Python CI
on:
    push:
        branches:
            - master
    pull_request:
    merge_group:

permissions:
    contents: read
    pull-requests: read

concurrency:
    group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
    cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
    # Job to decide if we should run python ci
    # See https://github.com/dorny/paths-filter#conditional-execution for more details
    changes:
        runs-on: ubuntu-latest
        timeout-minutes: 5
        name: Determine need to run python checks
        # Set job outputs to values from filter step
        outputs:
            python: ${{ steps.filter.outputs.python || 'true' }}
        steps:
            # For pull requests it's not necessary to checkout the code, but we
            # also want this to run on master so we need to checkout
            - uses: actions/checkout@v6

            - uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
              id: app-token
              if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
              with:
                  client-id: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_APP_ID }}
                  private-key: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_PRIVATE_KEY }}
            - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
              id: filter
              if: github.event_name != 'push' # Run all tests on master push
              with:
                  token: ${{ steps.app-token.outputs.token || github.token }}
                  filters: |
                      python:
                        - 'pyproject.toml'
                        - 'uv.lock'
                        - 'ee/**/*.py'
                        - 'posthog/**/*'
                        - 'products/**/*.py'
                        # Make sure we run if someone is explicitly change the workflows
                        - .github/workflows/ci-python.yml
                        - .github/workflows/ci-dagster.yml
                        - .github/workflows/ci-backend.yml
                        - .flox/env/manifest.toml
                        - bin/check_uv_python_compatibility.py
                        # Schema changes need to trigger Python CI to validate schema.py is regenerated
                        - 'frontend/src/queries/schema.json'
                        # Products changes need to trigger Python CI to validate the json usage is still valid
                        - 'frontend/src/products.json'
                        # hogli CLI changes need validation
                        - 'tools/hogli/**'
                        - 'tools/hogli-commands/hogli_commands/**'
                        - 'hogli.yaml'
                        - 'bin/**'
                        # LLM Gateway service
                        - 'services/llm-gateway/**/*.py'
                        # GitHub scripts
                        - '.github/scripts/**/*.py'
                        # PR approval agent
                        - 'tools/pr-approval-agent/**'

    code-quality:
        needs: changes
        if: needs.changes.outputs.python == 'true'
        timeout-minutes: 30

        strategy:
            fail-fast: false
            matrix:
                include: ${{ vars.BLACKSMITH_SHADOW_ENABLED == 'true' && fromJson('[{"runner":"depot-ubuntu-latest","is_shadow":false},{"runner":"blacksmith-2vcpu-ubuntu-2404","is_shadow":true}]') || fromJson('[{"runner":"depot-ubuntu-latest","is_shadow":false}]') }}
        name: Python code quality (${{ matrix.runner }})
        runs-on: ${{ matrix.runner }}
        continue-on-error: ${{ matrix.is_shadow }}

        steps:
            - uses: actions/checkout@v6
              with:
                  fetch-depth: 1

            - name: Set up Python
              uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
              with:
                  python-version: 3.12.12

            - name: Install uv
              id: setup-uv
              uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0
              with:
                  version: '0.10.2' # pinned: unpinned setup-uv calls GH API on every job, exhausts rate limit
                  enable-cache: true
                  cache-dependency-glob: uv.lock
                  save-cache: ${{ github.ref == 'refs/heads/master' }}

            - name: Check uv and Python version compatibility
              shell: bash
              run: |
                  python3 bin/check_uv_python_compatibility.py

            - name: Install SAML (python3-saml) dependencies
              if: steps.setup-uv.outputs.cache-hit != 'true'
              shell: bash
              run: |
                  sudo apt-get update
                  sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl

            - name: Install Python dependencies
              shell: bash
              run: |
                  UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev

            - name: Download GeoIP database
              shell: bash
              run: |
                  ./bin/download-mmdb

            - name: Check for syntax errors, import sort, and code style violations
              shell: bash
              run: |
                  ruff check .

            - name: Check formatting
              shell: bash
              run: |
                  ruff format --check --diff .

            - name: Add ty Problem Matcher
              shell: bash
              run: |
                  echo "::add-matcher::.github/ty-problem-matcher.json"

            - name: Check ty type checking
              shell: bash
              run: |
                  echo "📊 Feedback welcome in #team-devex"
                  uv run ty check

            - name: Add mypy Problem Matcher
              shell: bash
              run: |
                  echo "::add-matcher::.github/mypy-problem-matcher.json"

            - name: Restore mypy cache
              uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5
              with:
                  path: .mypy_cache
                  key: mypy-cache-${{ runner.os }}-3.12-${{ hashFiles('**/pyproject.toml', '**/mypy.ini') }}
                  restore-keys: |
                      mypy-cache-${{ runner.os }}-3.12-

            - name: Check static typing
              shell: bash -e {0}
              run: |
                  mypy --version && mypy --cache-fine-grained .

            - name: Save mypy cache
              # Shadow restores the saved cache for apples-to-apples timing.
              if: github.ref == 'refs/heads/master' && !matrix.is_shadow
              uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5
              with:
                  path: .mypy_cache
                  key: mypy-cache-${{ runner.os }}-3.12-${{ hashFiles('**/pyproject.toml', '**/mypy.ini') }}

            - name: Check if "schema.py" is up to date
              shell: bash
              run: |
                  npm run schema:build:python && git diff --exit-code

            - name: Check hogli manifest completeness
              shell: bash
              run: |
                  ./bin/hogli meta:check

            - name: Run hogli framework tests
              shell: bash
              run: |
                  pytest tools/hogli/tests -v --junitxml=junit-hogli.xml

            - name: Run hogli extension tests
              shell: bash
              run: |
                  pytest tools/hogli-commands/hogli_commands/tests -v --junitxml=junit-hogli-commands.xml

            - name: Upload test results
              uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
              if: always() && !matrix.is_shadow
              with:
                  name: junit-results-hogli
                  path: |
                      junit-hogli.xml
                      junit-hogli-commands.xml

            - name: Run dependency detection script tests
              shell: bash
              run: |
                  pytest bin/test/test_find_python_dependencies.py -v --junitxml=junit-dependency-detection.xml

            - name: Upload test results for dependency detection
              uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
              if: always() && !matrix.is_shadow
              with:
                  name: junit-results-dependency-detection
                  path: junit-dependency-detection.xml

            # - name: Check if "taxonomy.json/taxonomy.tsx" is up to date
            #   if: needs.changes.outputs.python == 'true'
            #   shell: bash
            #   run: |
            #       npm run taxonomy:build:json && git diff --exit-code

    # Collate job - single required status check for branch protection.
    # The code-quality job skips when no Python changes, but this job always
    # runs and reports success, allowing PRs to merge.
    python_tests:
        needs: [changes, code-quality]
        name: Python code quality checks
        runs-on: ubuntu-latest
        timeout-minutes: 5
        if: always()
        steps:
            - name: Check Python CI status
              run: |
                  # Fail if change detection itself failed
                  if [[ "${{ needs.changes.result }}" == "failure" ]]; then
                    echo "Change detection job failed."
                    exit 1
                  fi

                  # Pass if no Python changes detected (jobs were skipped)
                  if [[ "${{ needs.changes.outputs.python }}" != "true" ]]; then
                    echo "Python checks were skipped (no relevant changes)"
                    exit 0
                  fi

                  # Check actual job result
                  if [[ "${{ needs.code-quality.result }}" != "success" && "${{ needs.code-quality.result }}" != "skipped" ]]; then
                    echo "Python code quality checks failed."
                    exit 1
                  fi

                  echo "All Python checks passed."