chore: auto fetch include analysis output in more scripts if not provided

Author: dsanders11

include_analysis_diff.py

@@ -12,6 +12,7 @@
 from extract_archived_include_analysis import extract_include_analysis
 from include_analysis import IncludeAnalysisOutput, ParseError, parse_raw_include_analysis_output
 from suggest_include_changes import filter_filenames
+from utils import get_latest_include_analysis
 
 CHROMIUM_INCLUDE_ANALYSIS_BASE_URL = "https://commondatastorage.googleapis.com/chromium-browser-clang"
 HREF_REGEX = re.compile(r"<a href=\"(.*?)\">", re.DOTALL)
@@ -243,10 +244,7 @@ def main():
     if args.include_analysis_output:
         raw_include_analysis = args.include_analysis_output.read()
     else:
-        include_analysis_response = urllib.request.urlopen(
-            "https://commondatastorage.googleapis.com/chromium-browser-clang/include-analysis.js"
-        )
-        raw_include_analysis = include_analysis_response.read().decode("utf8")
+        raw_include_analysis = get_latest_include_analysis()
 
     try:
         include_analysis = parse_raw_include_analysis_output(raw_include_analysis)

list_includers.py

@@ -17,6 +17,7 @@
     get_include_analysis_edge_includer_size,
     get_include_analysis_edge_prevalence,
     get_include_analysis_edge_sizes,
+    get_latest_include_analysis,
     load_config,
     normalize_include_path,
 )
@@ -101,6 +102,7 @@ def main():
     parser.add_argument(
         "include_analysis_output",
         type=argparse.FileType("r"),
+        nargs="?",
         help="The include analysis output to use.",
     )
     parser.add_argument("filename", help="File to list includers for.")
@@ -126,8 +128,14 @@ def main():
     parser.add_argument("--verbose", action="store_true", default=False, help="Enable verbose logging.")
     args = parser.parse_args()
 
+    # If the user specified an include analysis output file, use that instead of fetching it
+    if args.include_analysis_output:
+        raw_include_analysis = args.include_analysis_output.read()
+    else:
+        raw_include_analysis = get_latest_include_analysis()
+
     try:
-        include_analysis = parse_raw_include_analysis_output(args.include_analysis_output.read())
+        include_analysis = parse_raw_include_analysis_output(raw_include_analysis)
     except ParseError as e:
         message = str(e)
         print("error: Could not parse include analysis output file")

list_transitive_includes.py

@@ -19,6 +19,7 @@
     get_include_analysis_edge_prevalence,
     get_include_analysis_edge_sizes,
     get_include_file_size,
+    get_latest_include_analysis,
     load_config,
     normalize_include_path,
 )
@@ -141,6 +142,7 @@ def main():
     parser.add_argument(
         "include_analysis_output",
         type=argparse.FileType("r"),
+        nargs="?",
         help="The include analysis output to use.",
     )
     parser.add_argument("filename", help="File to list includes for.")
@@ -177,8 +179,14 @@ def main():
         print("error: --apply-changes option requires --include-changes")
         return 1
 
+    # If the user specified an include analysis output file, use that instead of fetching it
+    if args.include_analysis_output:
+        raw_include_analysis = args.include_analysis_output.read()
+    else:
+        raw_include_analysis = get_latest_include_analysis()
+
     try:
-        include_analysis = parse_raw_include_analysis_output(args.include_analysis_output.read())
+        include_analysis = parse_raw_include_analysis_output(raw_include_analysis)
     except ParseError as e:
         message = str(e)
         print("error: Could not parse include analysis output file")

trace_transitive_include.py

@@ -11,7 +11,7 @@
 
 from include_analysis import IncludeAnalysisOutput, ParseError, parse_raw_include_analysis_output
 from typing import Iterator, Tuple
-from utils import create_graph_from_include_analysis
+from utils import create_graph_from_include_analysis, get_latest_include_analysis
 
 
 def trace_transitive_include(
@@ -33,15 +33,22 @@ def main():
     parser.add_argument(
         "include_analysis_output",
         type=argparse.FileType("r"),
+        nargs="?",
         help="The include analysis output to use.",
     )
     parser.add_argument("filename", help="File to start the trace from.")
     parser.add_argument("header", help="Target header to trace to.")
     parser.add_argument("--verbose", action="store_true", default=False, help="Enable verbose logging.")
     args = parser.parse_args()
 
+    # If the user specified an include analysis output file, use that instead of fetching it
+    if args.include_analysis_output:
+        raw_include_analysis = args.include_analysis_output.read()
+    else:
+        raw_include_analysis = get_latest_include_analysis()
+
     try:
-        include_analysis = parse_raw_include_analysis_output(args.include_analysis_output.read())
+        include_analysis = parse_raw_include_analysis_output(raw_include_analysis)
     except ParseError as e:
         message = str(e)
         print("error: Could not parse include analysis output file")

utils.py

@@ -3,7 +3,7 @@
 import multiprocessing
 import os
 import pathlib
-import re
+import urllib.request
 from collections import defaultdict
 from typing import DefaultDict, Dict, List
 
@@ -297,3 +297,11 @@ def normalize_include_path(
         _normalized_include_paths[include] = normalized
 
     return _normalized_include_paths[include]
+
+
+def get_latest_include_analysis():
+    response = urllib.request.urlopen(
+        "https://commondatastorage.googleapis.com/chromium-browser-clang/include-analysis.js"
+    )
+
+    return response.read().decode("utf8")