ci: workflow to surface heavy Chromium includes

Author: dsanders11

.github/workflows/include-analysis-edges.yml

@@ -0,0 +1,107 @@
+name: Heavy Chromium Includes
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 18 * * *'
+
+permissions: {}
+
+jobs:
+  heavy_chromium_includes:
+    name: Heavy Chromium Includes
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: dsanders11/chromium-include-cleanup
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+      - name: Install Dependencies
+        run: pip install -r requirements.txt
+      - name: Download Include Analysis Output
+        run: |
+          curl https://commondatastorage.googleapis.com/chromium-browser-clang/include-analysis.js > include-analysis.js
+      - name: Find Heavy Include edges
+        run: |
+          python extract_include_analysis_edges.py --filter-generated-files --filter-third-party --weight-threshold 75000000 > heavy-includes.csv
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: heavy-includes
+          path: heavy-includes.csv
+      - run: npm install @actions/cache
+      - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const fs = require('node:fs');
+            const cache = require('@actions/cache');
+
+            const data = fs.readFileSync('./heavy-includes.csv', 'utf8').trim();
+
+            const includes = await Promise.all(data.split('\n').map(async (line, idx) => {
+              const [filename, include, added_size, prevalence, expanded_size, centrality] = line.trim().split(',');
+              const addedSize = parseInt(added_size);
+
+              // Check if this is known from a previous run
+              const cacheKey = `heavy-chromium-include-${filename}-${include}`;
+              const cacheHit =
+                (await cache.restoreCache(['/dev/null'], cacheKey, undefined, {
+                  lookupOnly: true,
+                })) !== undefined;
+
+              if (!cacheHit) {
+                // Create a cache entry (only the name matters) to keep track of
+                // includes we've seen from previous runs to mark them as stale
+                await cache.saveCache(['/dev/null'], cacheKey);
+              }
+
+              return [
+                `${filename} --> ${include.replace(/</g, '&lt;').replace(/>/g, '&gt;')}`,
+                addedSize,
+                parseFloat(prevalenceData[idx].trim().split(',')[4]),
+                parseInt(expandedSizeData[idx].trim().split(',')[4]),
+                parseFloat(centralityData[idx].trim().split(',')[4]),
+                cacheHit,
+              ];
+            }));
+
+            const addTable = (includes) => {
+              core.summary.addTable([
+                [
+                  { data: 'Include Edge', header: true },
+                  { data: 'Added Size', header: true },
+                  { data: 'Prevalence', header: true },
+                  { data: 'Expanded Size', header: true },
+                  { data: 'Centrality', header: true },
+                ],
+                // Sort by added size, then convert it back to string or it won't render
+                ...includes
+                  .sort((a, b) => b[1] - a[1])
+                  .map(([edge, added_size, prevalence, expandedSize, centrality]) => [
+                    edge,
+                    added_size.toLocaleString(),
+                    `${prevalence.toFixed(2)}%`,
+                    expandedSize.toLocaleString(),
+                    centrality.toFixed(5),
+                  ]),
+              ]);
+            }
+
+            core.summary.addHeading('🔗 Heavy Chromium Includes');
+            core.summary.addRaw(`Found ${includes.length} unused includes over 75 MB`);
+
+            const newlySeen = unusedIncludes.filter(([, , , , , cacheHit]) => !cacheHit)
+            if (newlySeen.length > 0) {
+              core.summary.addHeading('Not Seen Before', '2');
+              addTable(newlySeen);
+              core.summary.addHeading('All Includes', '2');
+            } else {
+              core.summary.addBreak();
+            }
+
+            addTable(unusedIncludes);
+
+            await core.summary.write();

extract_include_analysis_edges.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+
+import argparse
+import csv
+import logging
+import os
+import re
+import sys
+import urllib.request
+from datetime import datetime
+
+from include_analysis import IncludeAnalysisOutput, ParseError, parse_raw_include_analysis_output
+from suggest_include_changes import filter_filenames
+from utils import (
+    get_include_analysis_edge_expanded_sizes,
+    get_include_analysis_edge_prevalence,
+    get_include_analysis_edges_centrality,
+    get_latest_include_analysis,
+)
+
+
+def extract_include_analysis_edges(
+    include_analysis: IncludeAnalysisOutput,
+    weight_threshold=None,
+    filter_generated_files=False,
+    filter_mojom_headers=False,
+    filter_third_party=False,
+):
+    filenames = filter_filenames(
+        include_analysis["files"],
+        filter_generated_files=filter_generated_files,
+        filter_mojom_headers=filter_mojom_headers,
+        filter_third_party=filter_third_party,
+    )
+
+    expanded_size_edge_weights = get_include_analysis_edge_expanded_sizes(include_analysis)
+    prevalence_edge_weights = get_include_analysis_edge_prevalence(include_analysis)
+    centrality_edge_weights = get_include_analysis_edges_centrality(include_analysis)
+
+    for file in filenames:
+        for [include, size] in include_analysis["esizes"][file].items():
+            if weight_threshold and float(size) < weight_threshold:
+                continue
+
+            prevalence = expanded_size_edge_weights[file][include]
+            expanded_size = expanded_size_edge_weights[file][include]
+            centrality = centrality_edge_weights[file][include]
+
+            yield file, include, size, prevalence, expanded_size, centrality
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Extract include edges from include analysis, with filtering")
+    parser.add_argument(
+        "include_analysis_output",
+        type=argparse.FileType("r"),
+        nargs="?",
+        help="The include analysis output to use.",
+    )
+    parser.add_argument(
+        "--weight-threshold", type=float, help="Filter out changes with a weight value below the threshold."
+    )
+    parser.add_argument(
+        "--filter-third-party", action="store_true", help="Filter out third_party/ (excluding blink) and v8."
+    )
+    parser.add_argument("--filter-generated-files", action="store_true", help="Filter out generated files.")
+    parser.add_argument("--filter-mojom-headers", action="store_true", help="Filter out mojom headers.")
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument("--quiet", action="store_true", default=False, help="Only log warnings and errors.")
+    group.add_argument("--verbose", action="store_true", default=False, help="Enable verbose logging.")
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(message)s",
+        level=logging.DEBUG if args.verbose else logging.WARNING if args.quiet else logging.INFO,
+    )
+
+    # If the user specified an include analysis output file, use that instead of fetching it
+    if args.include_analysis_output:
+        raw_include_analysis = args.include_analysis_output.read()
+    else:
+        raw_include_analysis = get_latest_include_analysis()
+
+    try:
+        include_analysis = parse_raw_include_analysis_output(raw_include_analysis)
+    except ParseError as e:
+        message = str(e)
+        print("error: Could not parse include analysis output file")
+        if message:
+            print(message)
+        return 2
+
+    csv_writer = csv.writer(sys.stdout)
+
+    try:
+        for row in extract_include_analysis_edges(
+            include_analysis,
+            weight_threshold=args.weight_threshold,
+            filter_generated_files=args.filter_generated_files,
+            filter_mojom_headers=args.filter_mojom_headers,
+            filter_third_party=args.filter_third_party,
+        ):
+            csv_writer.writerow(row)
+
+        sys.stdout.flush()
+    except BrokenPipeError:
+        devnull = os.open(os.devnull, os.O_WRONLY)
+        os.dup2(devnull, sys.stdout.fileno())
+        sys.exit(1)
+
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        pass  # Don't show the user anything