Support for reading in an apiview_properties.json file in the sources.jar file, for things like typespec back-referencing. (#7078)

Author: JonathanGiles

src/java/apiview-java-processor/src/main/java/com/azure/tools/apiview/processor/Main.java

@@ -3,11 +3,7 @@
 import com.azure.tools.apiview.processor.analysers.JavaASTAnalyser;
 import com.azure.tools.apiview.processor.analysers.Analyser;
 import com.azure.tools.apiview.processor.analysers.XMLASTAnalyser;
-import com.azure.tools.apiview.processor.model.APIListing;
-import com.azure.tools.apiview.processor.model.Diagnostic;
-import com.azure.tools.apiview.processor.model.DiagnosticKind;
-import com.azure.tools.apiview.processor.model.LanguageVariant;
-import com.azure.tools.apiview.processor.model.Token;
+import com.azure.tools.apiview.processor.model.*;
 import com.azure.tools.apiview.processor.model.maven.Pom;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -16,6 +12,7 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.net.URL;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
@@ -167,11 +164,26 @@ private static void processJavaSourcesJar(File inputFile, APIListing apiListing)
         }
         System.out.println("  Using '" + apiListing.getLanguageVariant() + "' for the language variant");
 
-        final Analyser analyser = new JavaASTAnalyser(inputFile, apiListing);
+        final Analyser analyser = new JavaASTAnalyser(apiListing);
 
         // Read all files within the jar file so that we can create a list of files to analyse
         final List<Path> allFiles = new ArrayList<>();
         try (FileSystem fs = FileSystems.newFileSystem(inputFile.toPath(), Main.class.getClassLoader())) {
+
+            try {
+                // we eagerly load the apiview_properties.json file into an ApiViewProperties object, so that it can
+                // be used throughout the analysis process, as required
+                URL apiViewPropertiesFile = fs.getPath("/META-INF/apiview_properties.json").toUri().toURL();
+                final ObjectMapper objectMapper = new ObjectMapper();
+                ApiViewProperties properties = objectMapper.readValue(apiViewPropertiesFile, ApiViewProperties.class);
+                apiListing.setApiViewProperties(properties);
+                System.out.println("  Found apiview_properties.json file in jar file");
+                System.out.println("    - Found " + properties.getCrossLanguageDefinitionIds().size() + " cross-language definition IDs");
+            } catch (Exception e) {
+                // this is fine, we just won't have any APIView properties to read in
+                System.out.println("  No apiview_properties.json file found in jar file - continuing...");
+            }
+
             fs.getRootDirectories().forEach(root -> {
                 try (Stream<Path> paths = Files.walk(root)) {
                     paths.forEach(allFiles::add);
@@ -183,6 +195,8 @@ private static void processJavaSourcesJar(File inputFile, APIListing apiListing)
 
             // Do the analysis while the filesystem is still represented in memory
             analyser.analyse(allFiles);
+        } catch (Exception e) {
+            e.printStackTrace();
         }
     }
 

src/java/apiview-java-processor/src/main/java/com/azure/tools/apiview/processor/analysers/JavaASTAnalyser.java

@@ -39,6 +39,7 @@
 import com.github.javaparser.ast.expr.NormalAnnotationExpr;
 import com.github.javaparser.ast.modules.ModuleDeclaration;
 import com.github.javaparser.ast.nodeTypes.NodeWithAnnotations;
+import com.github.javaparser.ast.nodeTypes.NodeWithSimpleName;
 import com.github.javaparser.ast.nodeTypes.NodeWithType;
 import com.github.javaparser.ast.type.ClassOrInterfaceType;
 import com.github.javaparser.ast.type.ReferenceType;
@@ -73,13 +74,7 @@
 
 import org.apache.commons.lang.StringEscapeUtils;
 
-import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.attemptToFindJavadocComment;
-import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.getPackageName;
-import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.isInterfaceType;
-import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.isPrivateOrPackagePrivate;
-import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.isPublicOrProtected;
-import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.isTypeAPublicAPI;
-import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.makeId;
+import static com.azure.tools.apiview.processor.analysers.util.ASTUtils.*;
 import static com.azure.tools.apiview.processor.analysers.util.TokenModifier.INDENT;
 import static com.azure.tools.apiview.processor.analysers.util.TokenModifier.NEWLINE;
 import static com.azure.tools.apiview.processor.analysers.util.TokenModifier.NOTHING;
@@ -111,19 +106,18 @@ public class JavaASTAnalyser implements Analyser {
 
     private static final Pattern SPLIT_NEWLINE = Pattern.compile(MiscUtils.LINEBREAK);
 
+    // This is the model that we build up as the AST of all files are analysed. The APIListing is then output as
+    // JSON that can be understood by APIView.
     private final APIListing apiListing;
 
-    private final Map<String, JavadocComment> packageNameToPackageInfoJavaDoc;
+    private final Map<String, JavadocComment> packageNameToPackageInfoJavaDoc = new HashMap<>();
 
-    private final Diagnostics diagnostic;
+    private final Diagnostics diagnostic = new Diagnostics();
 
-    private int indent;
+    private int indent = 0;
 
-    public JavaASTAnalyser(File inputFile, APIListing apiListing) {
+    public JavaASTAnalyser(APIListing apiListing) {
         this.apiListing = apiListing;
-        this.indent = 0;
-        this.packageNameToPackageInfoJavaDoc = new HashMap<>();
-        this.diagnostic = new Diagnostics();
     }
 
     @Override
@@ -183,10 +177,6 @@ public ScanClass(Path path, CompilationUnit compilationUnit) {
             }
         }
 
-        public CompilationUnit getCompilationUnit() {
-            return compilationUnit;
-        }
-
         public Path getPath() {
             return path;
         }
@@ -209,7 +199,6 @@ private Optional<ScanClass> scanForTypes(Path path) {
             // Set up a minimal type solver that only looks at the classes used to run this sample.
             CombinedTypeSolver combinedTypeSolver = new CombinedTypeSolver();
             combinedTypeSolver.add(new ReflectionTypeSolver(false));
-//            combinedTypeSolver.add(new SourceJarTypeSolver(inputFile));
 
             ParserConfiguration parserConfiguration = new ParserConfiguration()
                 .setStoreTokens(true)
@@ -343,7 +332,7 @@ private void tokeniseMavenPom(Pom mavenPom) {
                 // we don't care to present test scope dependencies
                 return;
             }
-            String scope = k.equals("") ? "compile" : k;
+            String scope = k.isEmpty() ? "compile" : k;
 
             addToken(makeWhitespace());
             addToken(new Token(COMMENT, "// " + scope + " scope"), NEWLINE);
@@ -510,6 +499,23 @@ private void visitModuleDeclaration(ModuleDeclaration moduleDeclaration) {
             addToken(new Token(TYPE_NAME, moduleDeclaration.getNameAsString(), MODULE_INFO_KEY), SPACE);
             addToken(new Token(PUNCTUATION, "{"), NEWLINE);
 
+            // Sometimes an exports or opens statement is conditional, so we need to handle that case
+            // in a single location here, to remove duplication.
+            Consumer<NodeList<Name>> conditionalExportsToOrOpensToConsumer = names -> {
+                if (!names.isEmpty()) {
+                    addToken(new Token(WHITESPACE, " "));
+                    addToken(new Token(KEYWORD, "to"), SPACE);
+
+                    for (int i = 0; i < names.size(); i++) {
+                        addToken(new Token(TYPE_NAME, names.get(i).toString()));
+
+                        if (i < names.size() - 1) {
+                            addToken(new Token(PUNCTUATION, ","), SPACE);
+                        }
+                    }
+                }
+            };
+
             moduleDeclaration.getDirectives().forEach(moduleDirective -> {
                 indent();
                 addToken(makeWhitespace());
@@ -532,43 +538,14 @@ private void visitModuleDeclaration(ModuleDeclaration moduleDeclaration) {
                 moduleDirective.ifModuleExportsStmt(d -> {
                     addToken(new Token(KEYWORD, "exports"), SPACE);
                     addToken(new Token(TYPE_NAME, d.getNameAsString(), makeId(MODULE_INFO_KEY + "-exports-" + d.getNameAsString())));
-
-                    NodeList<Name> names = d.getModuleNames();
-
-                    if (!names.isEmpty()) {
-                        addToken(new Token(WHITESPACE, " "));
-                        addToken(new Token(KEYWORD, "to"), SPACE);
-
-                        for (int i = 0; i < names.size(); i++) {
-                            addToken(new Token(TYPE_NAME, names.get(i).toString()));
-
-                            if (i < names.size() - 1) {
-                                addToken(new Token(PUNCTUATION, ","), SPACE);
-                            }
-                        }
-                    }
-
+                    conditionalExportsToOrOpensToConsumer.accept(d.getModuleNames());
                     addToken(new Token(PUNCTUATION, ";"), NEWLINE);
                 });
 
                 moduleDirective.ifModuleOpensStmt(d -> {
                     addToken(new Token(KEYWORD, "opens"), SPACE);
                     addToken(new Token(TYPE_NAME, d.getNameAsString(), makeId(MODULE_INFO_KEY + "-opens-" + d.getNameAsString())));
-
-                    NodeList<Name> names = d.getModuleNames();
-                    if (names.size() > 0) {
-                        addToken(new Token(WHITESPACE, " "));
-                        addToken(new Token(KEYWORD, "to"), SPACE);
-
-                        for (int i = 0; i < names.size(); i++) {
-                            addToken(new Token(TYPE_NAME, names.get(i).toString()));
-
-                            if (i < names.size() - 1) {
-                                addToken(new Token(PUNCTUATION, ","), SPACE);
-                            }
-                        }
-                    }
-
+                    conditionalExportsToOrOpensToConsumer.accept(d.getModuleNames());
                     addToken(new Token(PUNCTUATION, ";"), NEWLINE);
                 });
 
@@ -694,7 +671,13 @@ private void getTypeDeclaration(TypeDeclaration<?> typeDeclaration) {
                 addToken(new Token(DEPRECATED_RANGE_START));
             }
 
-            addToken(new Token(TYPE_NAME, className, classId));
+            // setting the class name. We need to look up to see if the apiview_properties.json file specified a
+            // cross language definition id for this type. If it did, we will use that. The apiview_properties.json
+            // file uses fully-qualified type names and method names, so we need to ensure that it what we are using
+            // when we look for a match.
+            Token typeNameToken = new Token(TYPE_NAME, className, classId);
+            checkForCrossLanguageDefinitionId(typeNameToken, typeDeclaration);
+            addToken(typeNameToken);
 
             if (isDeprecated) {
                 addToken(new Token(DEPRECATED_RANGE_END));
@@ -755,6 +738,25 @@ private void getTypeDeclaration(TypeDeclaration<?> typeDeclaration) {
             addToken(SPACE, new Token(PUNCTUATION, "{"), NEWLINE);
         }
 
+        /*
+         * This method is used to add 'cross language definition id' to the token if it is defined in the
+         * apiview_properties.json file. This is used most commonly in conjunction with TypeSpec-generated libraries,
+         * so that we may review cross languages with some level of confidence that the types and methods are the same.
+         */
+        private void checkForCrossLanguageDefinitionId(Token typeNameToken, NodeWithSimpleName<?> node) {
+            Optional<String> fqn;
+            if (node instanceof TypeDeclaration) {
+                fqn = ((TypeDeclaration<?>) node).getFullyQualifiedName();
+            } else if (node instanceof CallableDeclaration) {
+                fqn = Optional.of(getNodeFullyQualifiedName((CallableDeclaration<?>) node));
+            } else {
+                fqn = Optional.empty();
+            }
+
+            fqn.flatMap(_fqn -> apiListing.getApiViewProperties().getCrossLanguageDefinitionId(_fqn))
+               .ifPresent(typeNameToken::setCrossLanguageDefinitionId);
+        }
+
         private void tokeniseAnnotationMember(AnnotationDeclaration annotationDeclaration) {
             indent();
             // Member methods in the annotation declaration
@@ -1125,7 +1127,9 @@ private void getDeclarationNameAndParameters(CallableDeclaration<?> callableDecl
                 addToken(new Token(DEPRECATED_RANGE_START));
             }
 
-            addToken(new Token(MEMBER_NAME, name, definitionId));
+            Token nameToken = new Token(MEMBER_NAME, name, definitionId);
+            checkForCrossLanguageDefinitionId(nameToken, callableDeclaration);
+            addToken(nameToken);
 
             if (isDeprecated) {
                 addToken(new Token(DEPRECATED_RANGE_END));
@@ -1188,7 +1192,7 @@ private void getGenericTypeParameter(TypeParameter typeParameter) {
 
         private void getThrowException(CallableDeclaration<?> callableDeclaration) {
             final NodeList<ReferenceType> thrownExceptions = callableDeclaration.getThrownExceptions();
-            if (thrownExceptions.size() == 0) {
+            if (thrownExceptions.isEmpty()) {
                 return;
             }
 
@@ -1425,7 +1429,7 @@ private void visitJavaDoc(JavadocComment javadoc) {
                 // convert http/s links to external clickable links
                 Matcher urlMatch = MiscUtils.URL_MATCH.matcher(line2);
                 int currentIndex = 0;
-                while(urlMatch.find(currentIndex) == true) {
+                while(urlMatch.find(currentIndex)) {
                     int start = urlMatch.start();
                     int end = urlMatch.end();
 

src/java/apiview-java-processor/src/main/java/com/azure/tools/apiview/processor/analysers/util/ASTUtils.java

@@ -333,28 +333,37 @@ public static boolean isInterfaceType(TypeDeclaration<?> type) {
         return false;
     }
 
-    public static boolean isTypeImplementingInterface(TypeDeclaration type, String interfaceName) {
+    public static boolean isTypeImplementingInterface(TypeDeclaration<?> type, String interfaceName) {
         return type.asClassOrInterfaceDeclaration().getImplementedTypes().stream()
                 .anyMatch(_interface -> _interface.getNameAsString().equals(interfaceName));
     }
 
-    private static String getNodeFullyQualifiedName(Optional<Node> nodeOptional) {
-        if (!nodeOptional.isPresent()) {
+    public static String getNodeFullyQualifiedName(Node node) {
+        if (node == null) {
             return "";
         }
 
-        Node node = nodeOptional.get();
         if (node instanceof TypeDeclaration<?>) {
             TypeDeclaration<?> type = (TypeDeclaration<?>) node;
             return type.getFullyQualifiedName().get();
         } else if (node instanceof CallableDeclaration) {
-            CallableDeclaration callableDeclaration = (CallableDeclaration) node;
-            return getNodeFullyQualifiedName(node.getParentNode()) + "." + callableDeclaration.getNameAsString();
+            CallableDeclaration<?> callableDeclaration = (CallableDeclaration<?>) node;
+            String fqn = getNodeFullyQualifiedName(node.getParentNode()) + "." + callableDeclaration.getNameAsString();
+
+            if (callableDeclaration.isConstructorDeclaration()) {
+                fqn += ".ctor";
+            }
+
+            return fqn;
         } else {
             return "";
         }
     }
 
+    private static String getNodeFullyQualifiedName(Optional<Node> nodeOptional) {
+        return nodeOptional.map(ASTUtils::getNodeFullyQualifiedName).orElse("");
+    }
+
     /**
      * Attempts to retrieve the {@link JavadocComment} for a given {@link BodyDeclaration}.
      * <p>