Add benchmark workflow

sterliakov · sterliakov · commit 44c3d7a477ac · 2025-07-07T23:31:07.000+02:00
diff --git a/.github/workflows/check_pr.yaml b/.github/workflows/check_pr.yaml
@@ -6,6 +6,13 @@ on:
       pr:
         type: number
         description: PR to check
+        required: true
+      action:
+        type: choice
+        description: Action to perform
+        required: true
+        default: check
+        options: [check, bench]
       left_rev:
         type: string
         description: Left (old) revision
@@ -35,9 +42,10 @@ jobs:
     permissions:
       issues: write
     outputs:
-      pr-number: ${{ steps.pr-number.outputs.pr }}
+      pr_number: ${{ steps.pr-number.outputs.pr }}
+      action: ${{ steps.pr-number.outputs.action }}
     steps:
-      - name: Get Issue
+      - name: Get issue
         id: get-issue
         if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' }}
         uses: actions-cool/issues-helper@v3
@@ -46,22 +54,35 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Get PR number to check
-        if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' || inputs.pr != null }}
         id: pr-number
         run: |
-          if [ -z "${INPUT_PR}" ]; then
-              pr=$(sed -E 's/.*#([[:digit:]]+).*/\1/g' <<<"${ISSUE_TITLE}")
+          if [ -z "${INPUT_PR}" ] || [ -z "$ACTION" ]; then
+              case "${ISSUE_TITLE,,}" in
+                  'check #'*)
+                      pr=${ISSUE_TITLE##*#}
+                      action="check"
+                      ;;
+                  'bench #'*)
+                      pr=${ISSUE_TITLE##*#}
+                      action="bench"
+                      ;;
+                  *)
+                      pr="invalid"
+                      action="invalid"
+              esac
           else
               pr=${INPUT_PR}
+              action=${ACTION}
           fi
 
           re='^[0-9]+$'
           if ! [[ "$pr" =~ $re ]]; then
-              msg='No PR number found'
+              msg="No PR number found"
           elif ! gh --repo python/mypy pr view "$pr"; then
               msg=$(printf "PR #%s not found in python/mypy" "$pr")
           else
-              echo "pr=${pr}" >> "$GITHUB_OUTPUT"
+              printf "pr=%s" "${pr}" >> "$GITHUB_OUTPUT"
+              printf "action=%s" "${action}" >> "$GITHUB_OUTPUT"
               exit 0
           fi
           printf "%s\n" "$msg" >&2
@@ -70,6 +91,7 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }}
           INPUT_PR: ${{ inputs.pr }}
+          ACTION: ${{ inputs.action }}
           ISSUE_TITLE: ${{ steps.get-issue.outputs.issue-title }}
 
       - name: Report failure
@@ -83,9 +105,15 @@ jobs:
 
             Comment with any text to retry (feel free to edit the title beforehand).
 
+            Depending on the desider action, use the following issue titles:
+
+            * Check a PR against all open tickets: `Check #xxxxx`
+            * Run a few benchmarks for a PR: `Bench #xxxxx` or `Benchmark #xxxx`
+
   fetch:
     runs-on: ubuntu-latest
-    if: ${{ !github.event.issue.pull_request }}
+    needs: [get-target]
+    if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'check' }}
     steps:
       - uses: actions/cache/restore@v4
         id: restore-issues
@@ -120,7 +148,7 @@ jobs:
 
   apply:
     runs-on: ubuntu-latest
-    if: ${{ !github.event.issue.pull_request }}
+    if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'check' }}
     permissions:
       issues: write
     needs: [get-target, fetch]
@@ -152,7 +180,7 @@ jobs:
               --total-shards 2
         env:
           GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }}
-          PR: ${{ needs.get-target.outputs.pr-number }}
+          PR: ${{ needs.get-target.outputs.pr_number }}
           TQDM_MININTERVAL: '5'
 
       - name: Apply mypy (two versions)
@@ -190,7 +218,7 @@ jobs:
 
   diff:
     runs-on: ubuntu-latest
-    if: ${{ !github.event.issue.pull_request }}
+    if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'check' }}
     needs: [apply, get-target]
     permissions:
       issues: write
@@ -218,9 +246,9 @@ jobs:
         run: |
           uv run diff --no-snippets --diff-originals | tee out.txt
           {
-              echo 'diff-text<<EOF';
-              sed -e 's/\x1b\[1;3.m//g' -e 's/\x1b\[0m//g' out.txt;
-              echo 'EOF';
+              echo 'diff_text<<EOF'
+              sed -e 's/\x1b\[1;3.m//g' -e 's/\x1b\[0m//g' out.txt
+              echo 'EOF'
           } >> "$GITHUB_OUTPUT"
         env:
           GH_ACCESS_TOKEN: ${{ secrets.CUSTOM_GITHUB_PAT }}
@@ -232,12 +260,12 @@ jobs:
           actions: create-comment
           token: ${{ secrets.GITHUB_TOKEN }}
           body: |
-            Results are ready!
+            Check results are ready!
 
-            PR link: https://github.com/python/mypy/pull/${{ needs.get-target.outputs.pr-number }}
+            PR link: https://github.com/python/mypy/pull/${{ needs.get-target.outputs.pr_number }}
 
             ```diff
-            ${{ steps.compare.outputs.diff-text }}
+            ${{ steps.compare.outputs.diff_text }}
             ```
 
       - name: Report failure
@@ -248,3 +276,91 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           body: |
             Sorry, something went wrong during diff generation.
+
+  bench:
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'bench' }}
+    needs: [get-target]
+
+    strategy:
+      matrix:
+        iter: [1, 2, 3]
+    outputs:
+      result_run_1: ${{ steps.run-bench.outputs.result_run_1 }}
+      result_run_2: ${{ steps.run-bench.outputs.result_run_2 }}
+      result_run_3: ${{ steps.run-bench.outputs.result_run_3 }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          repository: python/mypy
+          token: ${{ secrets.CUSTOM_GITHUB_PAT }}
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: '3.12'
+
+      - name: Checkout PR
+        id: get-hash
+        run: |
+          gh pr checkout "$PR_NUMBER"
+          hash=$(git rev-parse HEAD)
+          printf 'target_hash=%s' "$hash" >>"$GITHUB_OUTPUT"
+        env:
+          PR_NUMBER: ${{ needs.get-target.outputs.pr_number }}
+
+      - name: Run benchmark
+        id: run-bench
+        run: |
+          uv pip install -e .
+          python ./misc/perf_compare.py master "$PR_HEAD_HASH" | tee results.txt
+          {
+              printf 'result_run_%s<<EOF' "$SEQUENCE_NO"
+              sed -ne '/===/,$ p' results.txt
+              echo EOF
+          } >>"$GITHUB_OUTPUT"
+        env:
+          PR_HEAD_HASH: ${{ steps.get-hash.outputs.target_hash }}
+          SEQUENCE_NO: ${{ matrix.iter }}
+
+      - name: Report failure
+        if: ${{ failure() && (github.event_name == 'issues' || github.event_name == 'issue_comment') }}
+        uses: actions-cool/issues-helper@v3
+        with:
+          actions: create-comment
+          token: ${{ secrets.GITHUB_TOKEN }}
+          body: |
+            Sorry, something went wrong during benchmark execution.
+
+  bench-report:
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.issue.pull_request && needs.get-target.outputs.action == 'bench' }}
+    needs: [get-target, bench]
+    permissions:
+      issues: write
+
+    steps:
+      - name: Post results
+        if: ${{ github.event_name == 'issues' || github.event_name == 'issue_comment' }}
+        uses: actions-cool/issues-helper@v3
+        with:
+          actions: create-comment
+          token: ${{ secrets.GITHUB_TOKEN }}
+          body: |
+            Benchmark results are ready!
+
+            PR link: https://github.com/python/mypy/pull/${{ needs.get-target.outputs.pr_number }}
+
+            Results over three rounds for selfcheck (N=15):
+
+            ```
+            ${{ needs.bench.outputs.result_run_1 }}
+            ```
+
+            ```
+            ${{ needs.bench.outputs.result_run_2 }}
+            ```
+
+            ```
+            ${{ needs.bench.outputs.result_run_3 }}
+            ```
diff --git a/src/mypy_issues/issues.py b/src/mypy_issues/issues.py
@@ -60,6 +60,7 @@ async def download_snippets(  # noqa: C901
     inventory: list[InventoryItem] = []
     issues: dict[int, IssueWithComments] = {}
     seen: set[str] = set()
+    last_known_issues: set[int] = set()
     removed_count = 0
     if INVENTORY_FILE.is_file() and ISSUES_FILE.is_file():
         inventory = load_inventory()
@@ -83,11 +84,11 @@ async def download_snippets(  # noqa: C901
         comments = await _get_comments(gh, org, repo, since)
         # Just sync issues with new comments from scratch, that should be cheap.
         removed.update(comments.keys())
+        last_known_issues = set(issues.keys())
         inventory, issues = _incremental_update(inventory, issues, removed)
         seen = {snip["filename"] for snip in inventory}
 
     event = asyncio.Event()
-    last_known_issues = set(issues.keys())
     new_issues: set[int] = set()
     async for batch in abatch(_get_issues(gh, event, org, repo, since), size=64):
         blocks = await asyncio.gather(*[
