check_pr.yaml

name: Check PR

on:
  workflow_dispatch:
    inputs:
      pr:
        type: number
        description: PR to check
      action:
        type: choice
        description: Action to perform
        required: true
        default: check
        options: [check, bench]
      left_rev:
        type: string
        description: Left (old) revision
      left_origin:
        type: string
        description: Left (old) origin
        default: pypi
      right_rev:
        type: string
        description: Right (new) revision
      right_origin:
        type: string
        description: Right (new) origin
        default: pypi
  issues:
    types: [opened, reopened]
  issue_comment:
    types: [created]

permissions:
  contents: read

env:
  BENCH_RUNS_SELFCHECK: 30
  BENCH_RUNS_SIMPLE: 50
  MARKER: <!-- marker -->

jobs:
  get-target:
    runs-on: ubuntu-latest
    if: ${{ !github.event.issue.pull_request }}
    permissions:
      issues: write
    outputs:
      pr_number: ${{ steps.pr-number.outputs.pr }}
      action: ${{ (github.event_name == 'issues' || github.event_name == 'issue_comment') && steps.pr-number.outputs.action || inputs.action }}
    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false

      - name: Get issue
        id: get-issue
        if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' }}
        uses: actions-cool/issues-helper@v3
        with:
          actions: get-issue
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Get PR number to check
        id: pr-number
        if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' }}
        run: |
          if [ -z "${INPUT_PR}" ] || [ -z "$ACTION" ]; then
              case "${ISSUE_TITLE,,}" in
                  'check #'*)
                      pr=${ISSUE_TITLE##*#}
                      action="check"
                      ;;
                  'bench #'*)
                      pr=${ISSUE_TITLE##*#}
                      action="bench"
                      ;;
                  *)
                      pr="invalid"
                      action="invalid"
              esac
          else
              pr=${INPUT_PR}
              action=${ACTION}
          fi

          re='^[0-9]+$'
          if ! [[ "$pr" =~ $re ]]; then
              msg="No PR number found"
          elif ! gh --repo python/mypy pr view "$pr"; then
              msg=$(printf "PR #%s not found in python/mypy" "$pr")
          else
              printf "pr=%s\n" "${pr}" >> "$GITHUB_OUTPUT"
              printf "action=%s\n" "${action}" >> "$GITHUB_OUTPUT"
              exit 0
          fi
          printf "%s\n" "$msg" >&2
          printf "msg=%s\n" "$msg" >> "$GITHUB_OUTPUT"
          exit 1
        env:
          GH_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }}
          INPUT_PR: ${{ inputs.pr }}
          ACTION: ${{ inputs.action }}
          ISSUE_TITLE: ${{ steps.get-issue.outputs.issue-title }}

      - name: Report progress
        if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' }}
        run: |
          gh issue view "$ISSUE" --json body | jq -r '.body' >body.txt
          cat >>body.txt <<-EOF
            ${{ env.MARKER }}

            Running $ACTION on #$INPUT_PR...
          EOF
          gh issue edit "$ISSUE" --body-file body.txt
          rm body.txt
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          INPUT_PR: ${{ steps.pr-number.outputs.pr }}
          ACTION: ${{ steps.pr-number.outputs.action }}
          ISSUE: ${{ github.event.issue.number }}

      - name: Report failure
        if: ${{ failure() && (github.event_name == 'issues' || github.event_name == 'issue_comment') }}
        uses: actions-cool/issues-helper@v3
        with:
          actions: create-comment
          token: ${{ secrets.GITHUB_TOKEN }}
          body: |
            ${{ steps.pr-number.outputs.msg }}

            Comment with any text to retry (feel free to edit the title beforehand).

            Depending on the desider action, use the following issue titles:

            * Check a PR against all open tickets: `Check #xxxxx`
            * Run a few benchmarks for a PR: `Bench #xxxxx`

  fetch:
    runs-on: ubuntu-latest
    needs: [get-target]
    if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'check' }}
    steps:
      - uses: actions/cache/restore@v5
        id: restore-issues
        name: Restore cache
        with:
          path: downloaded/
          key: issues-dump-v2-${{ github.run_id }}
          restore-keys: issues-dump-v2-

      - uses: actions/checkout@v6
        if: ${{ steps.restore-issues.outputs.cache-matched-key == '' }}
        with:
          persist-credentials: false
      - uses: astral-sh/setup-uv@v7
        if: ${{ steps.restore-issues.outputs.cache-matched-key == '' }}
        with:
          python-version: '3.13'

      - name: Fetch issues
            # cache-hit is false for partial matches
        if: ${{ steps.restore-issues.outputs.cache-matched-key == '' }}
        run: |
          uv run fetch
        env:
          GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }}

      - uses: actions/upload-artifact@v4
        with:
          name: issues-dump
          path: downloaded/
          retention-days: 1

  apply:
    runs-on: ubuntu-latest
    if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'check' }}
    permissions:
      issues: write
    needs: [get-target, fetch]
    strategy:
      matrix:
        side: [left, right]
        shard: [0, 1]

    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
      - uses: astral-sh/setup-uv@v7
        with:
          python-version: '3.13'

      - uses: actions/download-artifact@v5
        with:
          name: issues-dump
          path: downloaded/

      - name: Apply mypy (PR)
        if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' || inputs.pr != null }}
        run: |
          uv run run \
              --only-${{ matrix.side }} \
              --pr "${PR}" \
              --shard "${{ matrix.shard }}" \
              --total-shards 2
        env:
          GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }}
          PR: ${{ needs.get-target.outputs.pr_number }}
          TQDM_MININTERVAL: '5'

      - name: Apply mypy (two versions)
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.pr == null }}
        run: |
          uv run run \
              --only-${{ matrix.side }} \
              --left-rev "$LEFT_REV" \
              --left-origin "$LEFT_ORIGIN" \
              --right-rev "$RIGHT_REV" \
              --right-origin "$RIGHT_ORIGIN" \
              --shard "${{ matrix.shard }}" \
              --total-shards 2
        env:
          GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }}
          LEFT_REV: ${{ inputs.left_rev }}
          LEFT_ORIGIN: ${{ inputs.left_origin }}
          RIGHT_REV: ${{ inputs.right_rev }}
          RIGHT_ORIGIN: ${{ inputs.right_origin }}
          TQDM_MININTERVAL: '5'

      - uses: actions/upload-artifact@v4
        with:
          name: results-${{ matrix.shard }}-${{ matrix.side }}
          path: outputs/

      - name: Report failure
        if: ${{ failure() && (github.event_name == 'issues' || github.event_name == 'issue_comment') }}
        uses: actions-cool/issues-helper@v3
        with:
          actions: create-comment
          token: ${{ secrets.GITHUB_TOKEN }}
          body: |
            Sorry, something went wrong when running mypy.

  diff:
    runs-on: ubuntu-latest
    if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'check' }}
    needs: [apply, get-target]
    permissions:
      issues: write

    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
      - uses: astral-sh/setup-uv@v7
        with:
          python-version: '3.13'

      - uses: actions/download-artifact@v5
        with:
          name: issues-dump
          path: downloaded/
      - uses: actions/download-artifact@v5
        with:
          pattern: results-*
          path: outputs/
          merge-multiple: true

      - name: Compare outputs
        id: compare
        run: |
          uv run diff --no-snippets --diff-originals | tee out.txt
          {
              echo 'diff_text<<EOF'
              sed -e 's/\x1b\[1;3.m//g' -e 's/\x1b\[0m//g' out.txt
              echo 'EOF'
          } >> "$GITHUB_OUTPUT"
        env:
          GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }}

      - name: Post results
        if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' }}
        uses: actions-cool/issues-helper@v3
        with:
          actions: create-comment
          token: ${{ secrets.GITHUB_TOKEN }}
          body: |
            Check results are ready!

            PR link: https://github.com/python/mypy/pull/${{ needs.get-target.outputs.pr_number }}

            ```diff
            ${{ steps.compare.outputs.diff_text }}
            ```

      - name: Report failure
        if: ${{ failure() && (github.event_name == 'issues' || github.event_name == 'issue_comment') }}
        uses: actions-cool/issues-helper@v3
        with:
          actions: create-comment
          token: ${{ secrets.GITHUB_TOKEN }}
          body: |
            Sorry, something went wrong during diff generation.

  bench:
    runs-on: ubuntu-latest
    if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'bench' }}
    needs: [get-target]
    permissions:
      issues: write

    strategy:
      matrix:
        iter: [1, 2, 3]
        sample: [selfcheck, simple]
    outputs:
      result_run_selfcheck_1: ${{ steps.run-bench.outputs.result_run_selfcheck_1 }}
      result_run_selfcheck_2: ${{ steps.run-bench.outputs.result_run_selfcheck_2 }}
      result_run_selfcheck_3: ${{ steps.run-bench.outputs.result_run_selfcheck_3 }}
      result_run_simple_1: ${{ steps.run-bench.outputs.result_run_simple_1 }}
      result_run_simple_2: ${{ steps.run-bench.outputs.result_run_simple_2 }}
      result_run_simple_3: ${{ steps.run-bench.outputs.result_run_simple_3 }}

    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
          repository: python/mypy
          fetch-depth: 0  # We need to have history access for merging
          token: ${{ secrets.CUSTOM_GITHUB_PAT }}
      - uses: astral-sh/setup-uv@v7
        with:
          python-version: '3.12'

      - name: Checkout PR
        id: get-hash
        run: |
          git config --global advice.detachedHead false
          git config user.name "Never pushes"
          git config user.email "does@not.exist"
          gh pr checkout "$PR_NUMBER"
          info=$(gh --repo python/mypy pr view 19623 --json baseRefName,baseRefOid)
          base_branch=$(jq -r '.baseRefName' <<<"$info")
          git merge "origin/$base_branch" --no-edit
          hash=$(git rev-parse HEAD)
          printf 'head_hash=%s\n' "$hash" >>"$GITHUB_OUTPUT"
          printf 'base_branch=%s\n' "$base_branch" >>"$GITHUB_OUTPUT"
        env:
          PR_NUMBER: ${{ needs.get-target.outputs.pr_number }}
          GH_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }}

      - name: Run benchmark
        id: run-bench
        run: |
          set -o pipefail

          uv venv
          . .venv/bin/activate
          uv pip install '.[mypyc]'
          uv pip install -r build-requirements.txt
          uv pip install -r test-requirements.txt

          extra_args=()
          if [[ "$CHECK" = "simple" ]]; then
            extra_args+=( '-c' 'import typing' )
          fi
          python ./misc/perf_compare.py \
            "$PR_BASE_BRANCH" "$PR_HEAD_HASH" \
            --num-runs "$NUM_RUNS" \
            "${extra_args[@]}" \
            | tee results.txt
          {
              printf 'result_run_%s_%s<<EOF\n' "$CHECK" "$SEQUENCE_NO"
              sed -ne '/=== Results/,$ p' results.txt
              echo EOF
          } >>"$GITHUB_OUTPUT"
        env:
          PR_HEAD_HASH: ${{ steps.get-hash.outputs.head_hash }}
          PR_BASE_BRANCH: ${{ steps.get-hash.outputs.base_branch }}
          NUM_RUNS: ${{ matrix.sample == 'simple' && env.BENCH_RUNS_SIMPLE || env.BENCH_RUNS_SELFCHECK }}
          CHECK: ${{ matrix.sample }}
          SEQUENCE_NO: ${{ matrix.iter }}

      - name: Report failure
        if: ${{ failure() && (github.event_name == 'issues' || github.event_name == 'issue_comment') }}
        uses: actions-cool/issues-helper@v3
        with:
          actions: create-comment
          token: ${{ secrets.GITHUB_TOKEN }}
          body: |
            Sorry, something went wrong during benchmark execution.

  bench-report:
    runs-on: ubuntu-latest
    if: |
      !github.event.issue.pull_request
      && needs.get-target.outputs.action == 'bench'
      && (github.event_name == 'issues' || github.event_name == 'issue_comment')
    needs: [get-target, bench]
    permissions:
      issues: write

    steps:
      - name: Post results
        uses: actions-cool/issues-helper@v3
        with:
          actions: create-comment
          token: ${{ secrets.GITHUB_TOKEN }}
          body: |
            Benchmark results are ready!

            PR link: https://github.com/python/mypy/pull/${{ needs.get-target.outputs.pr_number }}

            Results over three rounds for selfcheck (N=${{ env.BENCH_RUNS_SELFCHECK }}):

            ```
            ${{ needs.bench.outputs.result_run_selfcheck_1 }}
            ```

            ```
            ${{ needs.bench.outputs.result_run_selfcheck_2 }}
            ```

            ```
            ${{ needs.bench.outputs.result_run_selfcheck_3 }}
            ```

            Results over three rounds for `import typing` (N=${{ env.BENCH_RUNS_SIMPLE }}):

            ```
            ${{ needs.bench.outputs.result_run_simple_1 }}
            ```

            ```
            ${{ needs.bench.outputs.result_run_simple_2 }}
            ```

            ```
            ${{ needs.bench.outputs.result_run_simple_3 }}
            ```

  clear-body:
    runs-on: ubuntu-latest
    name: Remove progress report from issue body
    if: |
      always() && !cancelled()
      && !github.event.issue.pull_request
      && (github.event_name == 'issues' || github.event_name == 'issue_comment')
    needs: [bench-report, diff]
    permissions:
      issues: write
    steps:
      - uses: actions/checkout@v6
        with:
          persist-credentials: false
      - name: Clear body
        run: |
          gh issue view "$ISSUE" --json body | jq -r '.body' >body.txt
          sed -e '/${{ env.MARKER }}/Q' body.txt >body.new.txt
          gh issue edit "$ISSUE" --body-file body.new.txt
          rm body.txt body.new.txt
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          ISSUE: ${{ github.event.issue.number }}