Enhance Rust API Parser with Stable Line IDs (#1)

* formattign changes and "use name = summary" pattern

* replaceCratePath utils

* split if glob for "use"

* checkpoint - do not show internal details such as generated::*

* external module reexports being handled better

* formatting changes and upgrade version

* refactor common logic - re-export refs

* processExternalReferences

* linedid utils

* checkpoint with a bug

* new strategy to isolate the Use items - draft

* simple Use item - id fix

* template looks great

* addExternalReferencesIfNotExists improvement

* added a case where use item has not been processed yet, so we process it.

* parent info missed

* format

* bug fix for identity

* generic args fix

* registerExternalItemReference

* changelog

* collapse rootModule into top-level

* replaceSuperPrefix

* line id utils added

* update function name

* stable lineIds updates

Author: HarshaNalluru

tools/apiview/parsers/rust-api-parser/src/main.ts

@@ -4,6 +4,7 @@ import { CodeFile, TokenKind } from "./models/apiview-models";
 import { Crate, FORMAT_VERSION, Id } from "../rustdoc-types/output/rustdoc-types";
 import { externalReferencesLines } from "./process-items/utils/externalReexports";
 import { sortExternalItems } from "./process-items/utils/sorting";
+import { updateReviewLinesWithStableLineIds } from "./utils/lineIdUtils";
 
 let apiJson: Crate;
 export const processedItems = new Set<number>();
@@ -77,6 +78,8 @@ function buildCodeFile(): CodeFile {
 
   processRootItem(codeFile);
   processExternalReferences(codeFile);
+
+  updateReviewLinesWithStableLineIds(codeFile.ReviewLines);
   return codeFile;
 }
 

tools/apiview/parsers/rust-api-parser/src/process-items/processAssocConst.ts

@@ -3,6 +3,7 @@ import { Item } from "../../rustdoc-types/output/rustdoc-types";
 import { createDocsReviewLines } from "./utils/generateDocReviewLine";
 import { typeToReviewTokens } from "./utils/typeToReviewTokens";
 import { isAssocConstItem } from "./utils/typeGuards";
+import { lineIdMap } from "../utils/lineIdUtils";
 
 /**
  * Processes an associated constant item and returns ReviewLine objects.
@@ -62,5 +63,6 @@ export function processAssocConst(item: Item): ReviewLine[] | null {
   });
 
   reviewLines.push(reviewLine);
+  lineIdMap.set(item.id.toString(), `const_${item.name}`);
   return reviewLines;
 }

tools/apiview/parsers/rust-api-parser/src/process-items/processAssocType.ts

@@ -4,6 +4,7 @@ import { createDocsReviewLines } from "./utils/generateDocReviewLine";
 import { typeToReviewTokens } from "./utils/typeToReviewTokens";
 import { isAssocTypeItem } from "./utils/typeGuards";
 import { createGenericBoundTokens, processGenerics } from "./utils/processGenerics";
+import { lineIdMap } from "../utils/lineIdUtils";
 
 /**
  * Processes an associated type item and returns ReviewLine objects.
@@ -65,5 +66,6 @@ export function processAssocType(item: Item): ReviewLine[] | null {
   });
 
   reviewLines.push(reviewLine);
+  lineIdMap.set(item.id.toString(), `type_${item.name}`);
   return reviewLines;
 }

tools/apiview/parsers/rust-api-parser/src/process-items/processConstant.ts

@@ -3,6 +3,7 @@ import { Item } from "../../rustdoc-types/output/rustdoc-types";
 import { createDocsReviewLines } from "./utils/generateDocReviewLine";
 import { isConstantItem } from "./utils/typeGuards";
 import { typeToReviewTokens } from "./utils/typeToReviewTokens";
+import { lineIdMap } from "../utils/lineIdUtils";
 
 export function processConstant(item: Item) {
   if (!isConstantItem(item)) return;
@@ -56,5 +57,6 @@ export function processConstant(item: Item) {
     });
   }
   reviewLines.push(reviewLine);
+  lineIdMap.set(item.id.toString(), `const_${item.name}`);
   return reviewLines;
 }

tools/apiview/parsers/rust-api-parser/src/process-items/processEnum.ts

@@ -5,12 +5,14 @@ import { createDocsReviewLines } from "./utils/generateDocReviewLine";
 import { processGenerics } from "./utils/processGenerics";
 import { isEnumItem } from "./utils/typeGuards";
 import { getAPIJson } from "../main";
+import { lineIdMap } from "../utils/lineIdUtils";
 
 export function processEnum(item: Item): ReviewLine[] {
   if (!isEnumItem(item)) return [];
   const apiJson = getAPIJson();
   const reviewLines: ReviewLine[] = item.docs ? createDocsReviewLines(item) : [];
 
+  lineIdMap.set(item.id.toString(), `enum_${item.name}`);
   // Process derives and impls
   let implResult: ImplProcessResult;
   if (item.inner.enum.impls) {
@@ -25,7 +27,6 @@ export function processEnum(item: Item): ReviewLine[] {
 
   if (implResult.deriveTokens.length > 0) {
     const deriveTokensLine: ReviewLine = {
-      LineId: item.id.toString() + "_derive",
       Tokens: implResult.deriveTokens,
       RelatedToLine: item.id.toString(),
     };
@@ -67,6 +68,7 @@ export function processEnum(item: Item): ReviewLine[] {
   if (item.inner.enum.variants) {
     enumLine.Children = item.inner.enum.variants.map((variant: number) => {
       const variantItem = apiJson.index[variant];
+      lineIdMap.set(variantItem.id.toString(), `variant_${variantItem.name}`);
       return {
         LineId: variantItem.id.toString(),
         Tokens: [
@@ -97,5 +99,6 @@ export function processEnum(item: Item): ReviewLine[] {
   if (implResult.traitImpls.length > 0) {
     reviewLines.push(...implResult.traitImpls);
   }
+  lineIdMap.set(item.id.toString(), `enum_${item.name}`);
   return reviewLines;
 }

tools/apiview/parsers/rust-api-parser/src/process-items/processExternType.ts

@@ -2,6 +2,7 @@ import { ReviewLine, TokenKind } from "../models/apiview-models";
 import { Item } from "../../rustdoc-types/output/rustdoc-types";
 import { createDocsReviewLines } from "./utils/generateDocReviewLine";
 import { isExternTypeItem } from "./utils/typeGuards";
+import { lineIdMap } from "../utils/lineIdUtils";
 
 /**
  * Processes an extern type item and returns ReviewLine objects.
@@ -40,5 +41,6 @@ export function processExternType(item: Item): ReviewLine[] | null {
   });
 
   reviewLines.push(reviewLine);
+  lineIdMap.set(item.id.toString(), `extern_type_${item.name}`);
   return reviewLines;
 }

tools/apiview/parsers/rust-api-parser/src/process-items/processFunction.ts

@@ -4,6 +4,7 @@ import { createDocsReviewLines } from "./utils/generateDocReviewLine";
 import { processGenerics } from "./utils/processGenerics";
 import { isFunctionItem } from "./utils/typeGuards";
 import { typeToReviewTokens } from "./utils/typeToReviewTokens";
+import { lineIdMap } from "../utils/lineIdUtils";
 
 /**
  * Processes the function header and adds modifiers and ABI information to the tokens
@@ -175,5 +176,6 @@ export function processFunction(item: Item): ReviewLine[] {
     });
   }
   reviewLines.push(reviewLine);
+  lineIdMap.set(item.id.toString(), `function_${item.name}`);
   return reviewLines;
 }

tools/apiview/parsers/rust-api-parser/src/process-items/processImpl.ts

@@ -12,6 +12,7 @@ import {
 import { typeToReviewTokens } from "./utils/typeToReviewTokens";
 import { processGenericArgs, processGenerics } from "./utils/processGenerics";
 import { getAPIJson } from "../main";
+import { lineIdMap } from "../utils/lineIdUtils";
 
 // Interface for the result of processing implementations
 export interface ImplProcessResult {
@@ -60,7 +61,7 @@ export function processAutoTraitImpls(impls: number[]): ReviewToken[] {
 }
 
 // Process manually implemented trait implementations
-function processOtherTraitImpls(impls: number[]): ReviewLine[] {
+function processOtherTraitImpls(impls: number[], prefixId: string): ReviewLine[] {
   const apiJson = getAPIJson();
   const traitImpls = getFilteredImpls(impls, isManualTraitImpl);
   return traitImpls.flatMap((implItem) => {
@@ -69,33 +70,35 @@ function processOtherTraitImpls(impls: number[]): ReviewLine[] {
     // Get the type that the trait is implemented for
     const parentName = typeToReviewTokens(implItem.inner.impl.for);
 
-    const implId = implItem.id.toString();
+    const lineId = implItem.id.toString() + "_" + prefixId;
     // Process provided trait methods that are mentioned but not explicitly implemented
     const providedTraitMethods: ReviewLine[] = implItem.inner.impl.provided_trait_methods.map(
-      (method) => ({
-        LineId: implId + "_impl_" + method,
-        Tokens: [
-          { Kind: TokenKind.Keyword, Value: "pub" },
-          { Kind: TokenKind.Keyword, Value: "fn" },
-          {
-            Kind: TokenKind.MemberName,
-            Value: method,
-            RenderClasses: ["method"],
-            HasSuffixSpace: false,
-          },
-          { Kind: TokenKind.Punctuation, Value: ";" },
-          {
-            Kind: TokenKind.Comment,
-            Value: "// provided trait method",
-            HasSuffixSpace: false,
-          },
-        ],
-      }),
+      (method) => {
+        return {
+          Tokens: [
+            { Kind: TokenKind.Keyword, Value: "pub" },
+            { Kind: TokenKind.Keyword, Value: "fn" },
+            {
+              Kind: TokenKind.MemberName,
+              Value: method,
+              RenderClasses: ["method"],
+              HasSuffixSpace: false,
+            },
+            { Kind: TokenKind.Punctuation, Value: ";" },
+            {
+              Kind: TokenKind.Comment,
+              Value: "// provided trait method",
+              HasSuffixSpace: false,
+            },
+          ],
+          RelatedToLine: lineId,
+        };
+      },
     );
     const implGenerics = processGenerics(implItem.inner.impl.generics);
     // Create the main impl line with trait name and type
     const reviewLineForImpl: ReviewLine = {
-      LineId: implId + "_impl",
+      LineId: lineId,
       Tokens: [
         {
           Kind: TokenKind.Keyword,
@@ -108,7 +111,7 @@ function processOtherTraitImpls(impls: number[]): ReviewLine[] {
           Value: (implItem.inner.impl.is_negative ? "!" : "") + implItem.inner.impl.trait.name,
           HasPrefixSpace: true,
           HasSuffixSpace: false,
-          NavigateToId: implId + "_impl",
+          NavigateToId: lineId,
           // Create navigation display name by combining trait and type names
           NavigationDisplayName:
             implItem.inner.impl.trait.name + "_" + parentName.map((token) => token.Value).join(""),
@@ -132,8 +135,16 @@ function processOtherTraitImpls(impls: number[]): ReviewLine[] {
       ],
     };
 
+    lineIdMap.set(
+      lineId,
+      prefixId +
+        reviewLineForImpl.Tokens.map((token) => token.Value)
+          .join("_")
+          .replace(/[^a-zA-Z0-9]+/g, ""),
+    );
+
     const closingLine: ReviewLine = {
-      RelatedToLine: implItem.id.toString() + "_impl",
+      RelatedToLine: lineId,
       Tokens: [{ Kind: TokenKind.Punctuation, Value: "}" }],
     };
 
@@ -166,6 +177,8 @@ export function processImpl(
     | (Item & { inner: { enum: Enum } })
     | (Item & { inner: { union: Union } }),
 ): ImplProcessResult {
+  const linedId = item.id.toString() + "_impl";
+  lineIdMap.set(linedId, lineIdMap.get(item.id.toString()) + "_impl");
   // Get all implementations associated with this item
   const impls = getImplsFromItem(item);
 
@@ -176,35 +189,35 @@ export function processImpl(
   // Process children first to check if they're empty
   const children = processImpls(impls).filter((item) => item != null);
 
-  // Only create an implBlock if there are children
-  const implBlock: ReviewLine[] =
-    children.length === 0
-      ? [] // Empty implBlock if no children
-      : [
-          // Create the main implementation line with type name
-          {
-            LineId: item.id.toString() + "_impl",
-            Tokens: [
-              { Kind: TokenKind.Keyword, Value: "impl" },
-              {
-                Kind: TokenKind.MemberName,
-                Value: item.name || "null",
-                RenderClasses: ["interface"],
-                NavigateToId: item.id.toString() + "_impl",
-                NavigationDisplayName: item.name || undefined,
-              },
-              { Kind: TokenKind.Punctuation, Value: "{" },
-            ],
-            Children: children,
-          },
+  let implBlock: ReviewLine[] = [];
+  if (children.length > 0) {
+    // Only create an implBlock if there are children
+    implBlock = [
+      // Create the main implementation line with type name
+      {
+        LineId: linedId,
+        Tokens: [
+          { Kind: TokenKind.Keyword, Value: "impl" },
           {
-            RelatedToLine: item.id.toString() + "_impl",
-            Tokens: [{ Kind: TokenKind.Punctuation, Value: "}" }],
+            Kind: TokenKind.MemberName,
+            Value: item.name || "null",
+            RenderClasses: ["interface"],
+            NavigateToId: linedId,
+            NavigationDisplayName: item.name || undefined,
           },
-        ];
+          { Kind: TokenKind.Punctuation, Value: "{" },
+        ],
+        Children: children,
+      },
+      {
+        RelatedToLine: linedId,
+        Tokens: [{ Kind: TokenKind.Punctuation, Value: "}" }],
+      },
+    ];
+  }
 
   // Process manual trait implementations (like impl Trait for Type { ... })
-  const traitImpls = processOtherTraitImpls(impls);
+  const traitImpls = processOtherTraitImpls(impls, item.name);
 
   return { deriveTokens, implBlock, traitImpls };
 }

tools/apiview/parsers/rust-api-parser/src/process-items/processMacro.ts

@@ -2,6 +2,7 @@ import { ReviewLine, TokenKind } from "../models/apiview-models";
 import { Item } from "../../rustdoc-types/output/rustdoc-types";
 import { createDocsReviewLines } from "./utils/generateDocReviewLine";
 import { isMacroItem } from "./utils/typeGuards";
+import { lineIdMap } from "../utils/lineIdUtils";
 
 /**
  * Processes a macro item and returns ReviewLine objects.
@@ -17,18 +18,20 @@ export function processMacro(item: Item): ReviewLine[] | null {
   // Split the macro value by newlines
   const macroLines = item.inner.macro.split("\n");
 
+  const linedId = item.id.toString();
+  lineIdMap.set(linedId, macroLines[0]);
   // Create ReviewLines for each macro line
   macroLines.forEach((line, index) => {
     const reviewLine: ReviewLine = {
-      LineId: index === 0 ? item.id.toString() : `${item.id.toString()}_macro_${index}`,
+      LineId: index === 0 ? linedId : undefined,
       Tokens: [
         {
           Kind: TokenKind.Text,
           Value: line,
         },
       ],
       // Only additional lines are related to the first line
-      RelatedToLine: index === 0 ? undefined : item.id.toString(),
+      RelatedToLine: index === 0 ? undefined : linedId,
     };
     reviewLines.push(reviewLine);
   });

tools/apiview/parsers/rust-api-parser/src/process-items/processModule.ts

@@ -7,6 +7,7 @@ import { getAPIJson, processedItems } from "../main";
 import { getSortedChildIds } from "./utils/sorting";
 import { processUse } from "./processUse";
 import { AnnotatedReviewLines } from "./utils/models";
+import { lineIdMap } from "../utils/lineIdUtils";
 
 /**
  * Processes a module item and adds its documentation to the ReviewLine.
@@ -24,6 +25,7 @@ export function processModule(
   const apiJson = getAPIJson();
   const isRootModule = item.id === apiJson.root;
 
+  lineIdMap.set(item.id.toString(), item.name || "null");
   // Create the ReviewLine object
   const reviewLine: ReviewLine = {
     LineId: item.id.toString(),
@@ -54,6 +56,7 @@ export function processModule(
       });
     }
     fullName += item.name;
+    lineIdMap.set(item.id.toString(), fullName);
     // current module
     reviewLine.Tokens.push({
       Kind: TokenKind.TypeName,