Check PR #112
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Find PR diffs | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pr: | |
| type: number | |
| description: PR to check | |
| required: true | |
| action: | |
| type: choice | |
| description: Action to perform | |
| required: true | |
| default: check | |
| options: [check, bench] | |
| left_rev: | |
| type: string | |
| description: Left (old) revision | |
| left_origin: | |
| type: string | |
| description: Left (old) origin | |
| default: pypi | |
| right_rev: | |
| type: string | |
| description: Right (new) revision | |
| right_origin: | |
| type: string | |
| description: Right (new) origin | |
| default: pypi | |
| issues: | |
| types: [opened, reopened] | |
| issue_comment: | |
| types: [created] | |
| permissions: | |
| contents: read | |
| jobs: | |
| get-target: | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.issue.pull_request }} | |
| permissions: | |
| issues: write | |
| outputs: | |
| pr_number: ${{ steps.pr-number.outputs.pr }} | |
| action: ${{ steps.pr-number.outputs.action }} | |
| steps: | |
| - name: Get issue | |
| id: get-issue | |
| if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: get-issue | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Get PR number to check | |
| id: pr-number | |
| run: | | |
| if [ -z "${INPUT_PR}" ] || [ -z "$ACTION" ]; then | |
| case "${ISSUE_TITLE,,}" in | |
| 'check #'*) | |
| pr=${ISSUE_TITLE##*#} | |
| action="check" | |
| ;; | |
| 'bench #'*) | |
| pr=${ISSUE_TITLE##*#} | |
| action="bench" | |
| ;; | |
| *) | |
| pr="invalid" | |
| action="invalid" | |
| esac | |
| else | |
| pr=${INPUT_PR} | |
| action=${ACTION} | |
| fi | |
| re='^[0-9]+$' | |
| if ! [[ "$pr" =~ $re ]]; then | |
| msg="No PR number found" | |
| elif ! gh --repo python/mypy pr view "$pr"; then | |
| msg=$(printf "PR #%s not found in python/mypy" "$pr") | |
| else | |
| printf "pr=%s" "${pr}" >> "$GITHUB_OUTPUT" | |
| printf "action=%s" "${action}" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| printf "%s\n" "$msg" >&2 | |
| printf "msg=%s\n" "$msg" >> "$GITHUB_OUTPUT" | |
| exit 1 | |
| env: | |
| GH_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }} | |
| INPUT_PR: ${{ inputs.pr }} | |
| ACTION: ${{ inputs.action }} | |
| ISSUE_TITLE: ${{ steps.get-issue.outputs.issue-title }} | |
| - name: Report failure | |
| if: ${{ failure() && (github.event_name == 'issues' || github.event_name == 'issue_comment') }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: create-comment | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| ${{ steps.pr-number.outputs.msg }} | |
| Comment with any text to retry (feel free to edit the title beforehand). | |
| Depending on the desider action, use the following issue titles: | |
| * Check a PR against all open tickets: `Check #xxxxx` | |
| * Run a few benchmarks for a PR: `Bench #xxxxx` or `Benchmark #xxxx` | |
| fetch: | |
| runs-on: ubuntu-latest | |
| needs: [get-target] | |
| if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'check' }} | |
| steps: | |
| - uses: actions/cache/restore@v4 | |
| id: restore-issues | |
| name: Restore cache | |
| with: | |
| path: downloaded/ | |
| key: issues-dump-v2-${{ github.run_id }} | |
| restore-keys: issues-dump-v2- | |
| - uses: actions/checkout@v4 | |
| if: ${{ steps.restore-issues.outputs.cache-matched-key == '' }} | |
| with: | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@v6 | |
| if: ${{ steps.restore-issues.outputs.cache-matched-key == '' }} | |
| with: | |
| python-version: '3.13' | |
| - name: Fetch issues | |
| # cache-hit is false for partial matches | |
| if: ${{ steps.restore-issues.outputs.cache-matched-key == '' }} | |
| run: | | |
| uv run fetch | |
| env: | |
| GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }} | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: issues-dump | |
| path: downloaded/ | |
| retention-days: 1 | |
| apply: | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'check' }} | |
| permissions: | |
| issues: write | |
| needs: [get-target, fetch] | |
| strategy: | |
| matrix: | |
| side: [left, right] | |
| shard: [0, 1] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@v6 | |
| with: | |
| python-version: '3.13' | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: issues-dump | |
| path: downloaded/ | |
| - name: Apply mypy (PR) | |
| if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' || inputs.pr != null }} | |
| run: | | |
| uv run run \ | |
| --only-${{ matrix.side }} \ | |
| --pr "${PR}" \ | |
| --shard "${{ matrix.shard }}" \ | |
| --total-shards 2 | |
| env: | |
| GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }} | |
| PR: ${{ needs.get-target.outputs.pr_number }} | |
| TQDM_MININTERVAL: '5' | |
| - name: Apply mypy (two versions) | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.pr == null }} | |
| run: | | |
| uv run run \ | |
| --only-${{ matrix.side }} \ | |
| --left-rev "$LEFT_REV" \ | |
| --left-origin "$LEFT_ORIGIN" \ | |
| --right-rev "$RIGHT_REV" \ | |
| --right-origin "$RIGHT_ORIGIN" \ | |
| --shard "${{ matrix.shard }}" \ | |
| --total-shards 2 | |
| env: | |
| GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }} | |
| LEFT_REV: ${{ inputs.left_rev }} | |
| LEFT_ORIGIN: ${{ inputs.left_origin }} | |
| RIGHT_REV: ${{ inputs.right_rev }} | |
| RIGHT_ORIGIN: ${{ inputs.right_origin }} | |
| TQDM_MININTERVAL: '5' | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-${{ matrix.shard }}-${{ matrix.side }} | |
| path: outputs/ | |
| - name: Report failure | |
| if: ${{ failure() && (github.event_name == 'issues' || github.event_name == 'issue_comment') }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: create-comment | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| Sorry, something went wrong when running mypy. | |
| diff: | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'check' }} | |
| needs: [apply, get-target] | |
| permissions: | |
| issues: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@v6 | |
| with: | |
| python-version: '3.13' | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: issues-dump | |
| path: downloaded/ | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: results-* | |
| path: outputs/ | |
| merge-multiple: true | |
| - name: Compare outputs | |
| id: compare | |
| run: | | |
| uv run diff --no-snippets --diff-originals | tee out.txt | |
| { | |
| echo 'diff_text<<EOF' | |
| sed -e 's/\x1b\[1;3.m//g' -e 's/\x1b\[0m//g' out.txt | |
| echo 'EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| env: | |
| GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }} | |
| - name: Post results | |
| if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: create-comment | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| Check results are ready! | |
| PR link: https://github.com/python/mypy/pull/${{ needs.get-target.outputs.pr_number }} | |
| ```diff | |
| ${{ steps.compare.outputs.diff_text }} | |
| ``` | |
| - name: Report failure | |
| if: ${{ failure() && (github.event_name == 'issues' || github.event_name == 'issue_comment') }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: create-comment | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| Sorry, something went wrong during diff generation. | |
| bench: | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'bench' }} | |
| needs: [get-target] | |
| strategy: | |
| matrix: | |
| iter: [1, 2, 3] | |
| outputs: | |
| result_run_1: ${{ steps.run-bench.outputs.result_run_1 }} | |
| result_run_2: ${{ steps.run-bench.outputs.result_run_2 }} | |
| result_run_3: ${{ steps.run-bench.outputs.result_run_3 }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| repository: python/mypy | |
| token: ${{ secrets.CUSTOM_GITHUB_PAT }} | |
| - uses: astral-sh/setup-uv@v6 | |
| with: | |
| python-version: '3.12' | |
| - name: Checkout PR | |
| id: get-hash | |
| run: | | |
| gh pr checkout "$PR_NUMBER" | |
| hash=$(git rev-parse HEAD) | |
| printf 'target_hash=%s' "$hash" >>"$GITHUB_OUTPUT" | |
| env: | |
| PR_NUMBER: ${{ needs.get-target.outputs.pr_number }} | |
| - name: Run benchmark | |
| id: run-bench | |
| run: | | |
| uv pip install -e . | |
| python ./misc/perf_compare.py master "$PR_HEAD_HASH" | tee results.txt | |
| { | |
| printf 'result_run_%s<<EOF' "$SEQUENCE_NO" | |
| sed -ne '/===/,$ p' results.txt | |
| echo EOF | |
| } >>"$GITHUB_OUTPUT" | |
| env: | |
| PR_HEAD_HASH: ${{ steps.get-hash.outputs.target_hash }} | |
| SEQUENCE_NO: ${{ matrix.iter }} | |
| - name: Report failure | |
| if: ${{ failure() && (github.event_name == 'issues' || github.event_name == 'issue_comment') }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: create-comment | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| Sorry, something went wrong during benchmark execution. | |
| bench-report: | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'bench' }} | |
| needs: [get-target, bench] | |
| permissions: | |
| issues: write | |
| steps: | |
| - name: Post results | |
| if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: create-comment | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| Benchmark results are ready! | |
| PR link: https://github.com/python/mypy/pull/${{ needs.get-target.outputs.pr_number }} | |
| Results over three rounds for selfcheck (N=15): | |
| ``` | |
| ${{ needs.bench.outputs.result_run_1 }} | |
| ``` | |
| ``` | |
| ${{ needs.bench.outputs.result_run_2 }} | |
| ``` | |
| ``` | |
| ${{ needs.bench.outputs.result_run_3 }} | |
| ``` |