Completion schema URI in modeline comment (#573)

* Provide completion for schema ids in modeline #560

potential improvements:
- support range when there is already a value provided
- provide completion for local yaml schema files

Extracted a specific file for modeline related things.

fixes #560

Signed-off-by: Aurélien Pupier <apupier@redhat.com>

* Add completion for schema URI im modeline comment

Signed-off-by: Yevhen Vydolob <yvydolob@redhat.com>

Co-authored-by: Aurélien Pupier <apupier@redhat.com>

Author: evidolob

src/languageserver/handlers/settingsHandlers.ts

@@ -195,6 +195,8 @@ export class SettingsHandler {
                 uri: schema.url,
                 fileMatch: [currFileMatch],
                 priority: SchemaPriority.SchemaStore,
+                name: schema.name,
+                description: schema.description,
               });
             }
           }

src/languageservice/parser/yaml-documents.ts

@@ -16,6 +16,7 @@ import { isArrayEqual } from '../utils/arrUtils';
 import { getParent } from '../utils/astUtils';
 import { TextBuffer } from '../utils/textBuffer';
 import { getIndentation } from '../utils/strings';
+import { Token } from 'yaml/dist/parse/cst';
 
 /**
  * These documents are collected into a final YAMLDocument
@@ -177,12 +178,15 @@ export class SingleYAMLDocument extends JSONDocument {
  * to the `parseYAML` caller.
  */
 export class YAMLDocument {
-  public documents: SingleYAMLDocument[];
+  documents: SingleYAMLDocument[];
+  tokens: Token[];
+
   private errors: YAMLDocDiagnostic[];
   private warnings: YAMLDocDiagnostic[];
 
-  constructor(documents: SingleYAMLDocument[]) {
+  constructor(documents: SingleYAMLDocument[], tokens: Token[]) {
     this.documents = documents;
+    this.tokens = tokens;
     this.errors = [];
     this.warnings = [];
   }
@@ -219,7 +223,7 @@ export class YamlDocuments {
   private ensureCache(document: TextDocument, parserOptions: ParserOptions, addRootObject: boolean): void {
     const key = document.uri;
     if (!this.cache.has(key)) {
-      this.cache.set(key, { version: -1, document: new YAMLDocument([]), parserOptions: defaultOptions });
+      this.cache.set(key, { version: -1, document: new YAMLDocument([], []), parserOptions: defaultOptions });
     }
     const cacheEntry = this.cache.get(key);
     if (

src/languageservice/parser/yamlParser07.ts

@@ -35,13 +35,13 @@ export function parse(text: string, parserOptions: ParserOptions = defaultOption
   const lineCounter = new LineCounter();
   const parser = new Parser(lineCounter.addNewLine);
   const tokens = parser.parse(text);
-  const docs = composer.compose(tokens, true);
-
+  const tokensArr = Array.from(tokens);
+  const docs = composer.compose(tokensArr, true, text.length);
   // Generate the SingleYAMLDocs from the AST nodes
   const yamlDocs: SingleYAMLDocument[] = Array.from(docs, (doc) => parsedDocToSingleYAMLDocument(doc, lineCounter));
 
   // Consolidate the SingleYAMLDocs
-  return new YAMLDocument(yamlDocs);
+  return new YAMLDocument(yamlDocs, tokensArr);
 }
 
 function parsedDocToSingleYAMLDocument(parsedDoc: Document, lineCounter: LineCounter): SingleYAMLDocument {

src/languageservice/services/modelineUtil.ts

@@ -0,0 +1,37 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Red Hat, Inc. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+import { SingleYAMLDocument } from '../parser/yamlParser07';
+import { JSONDocument } from '../parser/jsonParser07';
+
+/**
+ * Retrieve schema if declared as modeline.
+ * Public for testing purpose, not part of the API.
+ * @param doc
+ */
+export function getSchemaFromModeline(doc: SingleYAMLDocument | JSONDocument): string {
+  if (doc instanceof SingleYAMLDocument) {
+    const yamlLanguageServerModeline = doc.lineComments.find((lineComment) => {
+      return isModeline(lineComment);
+    });
+    if (yamlLanguageServerModeline != undefined) {
+      const schemaMatchs = yamlLanguageServerModeline.match(/\$schema=\S+/g);
+      if (schemaMatchs !== null && schemaMatchs.length >= 1) {
+        if (schemaMatchs.length >= 2) {
+          console.log(
+            'Several $schema attributes have been found on the yaml-language-server modeline. The first one will be picked.'
+          );
+        }
+        return schemaMatchs[0].substring('$schema='.length);
+      }
+    }
+  }
+  return undefined;
+}
+
+export function isModeline(lineText: string): boolean {
+  const matchModeline = lineText.match(/^#\s+yaml-language-server\s*:/g);
+  return matchModeline !== null && matchModeline.length === 1;
+}

src/languageservice/services/yamlCompletion.ts

@@ -10,6 +10,7 @@ import {
   CompletionItemKind,
   CompletionList,
   InsertTextFormat,
+  InsertTextMode,
   MarkupContent,
   MarkupKind,
   Position,
@@ -31,8 +32,9 @@ import { stringifyObject, StringifySettings } from '../utils/json';
 import { isDefined, isString } from '../utils/objects';
 import * as nls from 'vscode-nls';
 import { setKubernetesParserOption } from '../parser/isKubernetes';
-import { isMapContainsEmptyPair } from '../utils/astUtils';
+import { isInComment, isMapContainsEmptyPair } from '../utils/astUtils';
 import { indexOf } from '../utils/astUtils';
+import { isModeline } from './modelineUtil';
 
 const localize = nls.loadMessageBundle();
 
@@ -172,10 +174,16 @@ export class YamlCompletion {
       this.getCustomTagValueCompletions(collector);
     }
 
+    let lineContent = textBuffer.getLineContent(position.line);
+    if (lineContent.endsWith('\n')) {
+      lineContent = lineContent.substr(0, lineContent.length - 1);
+    }
+
     try {
       const schema = await this.schemaService.getSchemaForResource(document.uri, currentDoc);
+
       if (!schema || schema.errors.length) {
-        if (position.line === 0 && position.character === 0 && !textBuffer.getLineContent(0).includes('# yaml-language-server')) {
+        if (position.line === 0 && position.character === 0 && !isModeline(lineContent)) {
           const inlineSchemaCompletion = {
             kind: CompletionItemKind.Text,
             label: 'Inline schema',
@@ -184,6 +192,27 @@ export class YamlCompletion {
           };
           result.items.push(inlineSchemaCompletion);
         }
+      }
+
+      if (isModeline(lineContent) || isInComment(doc.tokens, offset)) {
+        const schemaIndex = lineContent.indexOf('$schema=');
+        if (schemaIndex !== -1 && schemaIndex + '$schema='.length <= position.character) {
+          this.schemaService.getAllSchemas().forEach((schema) => {
+            const schemaIdCompletion: CompletionItem = {
+              kind: CompletionItemKind.Constant,
+              label: schema.name ?? schema.uri,
+              detail: schema.description,
+              insertText: schema.uri,
+              insertTextFormat: InsertTextFormat.PlainText,
+              insertTextMode: InsertTextMode.asIs,
+            };
+            result.items.push(schemaIdCompletion);
+          });
+        }
+        return result;
+      }
+
+      if (!schema || schema.errors.length) {
         return result;
       }
 
@@ -203,10 +232,6 @@ export class YamlCompletion {
         }
       }
 
-      let lineContent = textBuffer.getLineContent(position.line);
-      if (lineContent.endsWith('\n')) {
-        lineContent = lineContent.substr(0, lineContent.length - 1);
-      }
       if (node) {
         if (lineContent.length === 0) {
           node = currentDoc.internalDocument.contents as Node;

src/languageservice/services/yamlSchemaService.ts

@@ -24,6 +24,8 @@ import { SingleYAMLDocument } from '../parser/yamlParser07';
 import { JSONDocument } from '../parser/jsonParser07';
 import { parse } from 'yaml';
 import * as path from 'path';
+import { getSchemaFromModeline } from './modelineUtil';
+import { JSONSchemaDescriptionExt } from '../../requestTypes';
 
 const localize = nls.loadMessageBundle();
 
@@ -94,6 +96,8 @@ export class YAMLSchemaService extends JSONSchemaService {
   private requestService: SchemaRequestService;
   public schemaPriorityMapping: Map<string, Set<SchemaPriority>>;
 
+  private schemaUriToNameAndDescription = new Map<string, [string, string]>();
+
   constructor(
     requestService: SchemaRequestService,
     contextService?: WorkspaceContextService,
@@ -109,6 +113,33 @@ export class YAMLSchemaService extends JSONSchemaService {
     this.customSchemaProvider = customSchemaProvider;
   }
 
+  getAllSchemas(): JSONSchemaDescriptionExt[] {
+    const result: JSONSchemaDescriptionExt[] = [];
+    const schemaUris = new Set<string>();
+    for (const filePattern of this.filePatternAssociations) {
+      const schemaUri = filePattern.uris[0];
+      if (schemaUris.has(schemaUri)) {
+        continue;
+      }
+      schemaUris.add(schemaUri);
+      const schemaHandle: JSONSchemaDescriptionExt = {
+        uri: schemaUri,
+        fromStore: false,
+        usedForCurrentFile: false,
+      };
+
+      if (this.schemaUriToNameAndDescription.has(schemaUri)) {
+        const [name, description] = this.schemaUriToNameAndDescription.get(schemaUri);
+        schemaHandle.name = name;
+        schemaHandle.description = description;
+        schemaHandle.fromStore = true;
+      }
+      result.push(schemaHandle);
+    }
+
+    return result;
+  }
+
   async resolveSchemaContent(
     schemaToResolve: UnresolvedSchema,
     schemaURL: string,
@@ -294,7 +325,7 @@ export class YAMLSchemaService extends JSONSchemaService {
       const seen: { [schemaId: string]: boolean } = Object.create(null);
       const schemas: string[] = [];
 
-      let schemaFromModeline = this.getSchemaFromModeline(doc);
+      let schemaFromModeline = getSchemaFromModeline(doc);
       if (schemaFromModeline !== undefined) {
         if (!schemaFromModeline.startsWith('file:') && !schemaFromModeline.startsWith('http')) {
           if (!path.isAbsolute(schemaFromModeline)) {
@@ -438,32 +469,6 @@ export class YAMLSchemaService extends JSONSchemaService {
     return priorityMapping.get(highestPrio) || [];
   }
 
-  /**
-   * Retrieve schema if declared as modeline.
-   * Public for testing purpose, not part of the API.
-   * @param doc
-   */
-  public getSchemaFromModeline(doc: SingleYAMLDocument | JSONDocument): string {
-    if (doc instanceof SingleYAMLDocument) {
-      const yamlLanguageServerModeline = doc.lineComments.find((lineComment) => {
-        const matchModeline = lineComment.match(/^#\s+yaml-language-server\s*:/g);
-        return matchModeline !== null && matchModeline.length === 1;
-      });
-      if (yamlLanguageServerModeline != undefined) {
-        const schemaMatchs = yamlLanguageServerModeline.match(/\$schema=\S+/g);
-        if (schemaMatchs !== null && schemaMatchs.length >= 1) {
-          if (schemaMatchs.length >= 2) {
-            console.log(
-              'Several $schema attributes have been found on the yaml-language-server modeline. The first one will be picked.'
-            );
-          }
-          return schemaMatchs[0].substring('$schema='.length);
-        }
-      }
-    }
-    return undefined;
-  }
-
   private async resolveCustomSchema(schemaUri, doc): ResolvedSchema {
     const unresolvedSchema = await this.loadSchema(schemaUri);
     const schema = await this.resolveSchemaContent(unresolvedSchema, schemaUri, []);
@@ -635,11 +640,25 @@ export class YAMLSchemaService extends JSONSchemaService {
         );
       }
       unresolvedJsonSchema.uri = schemaUri;
+      if (this.schemaUriToNameAndDescription.has(schemaUri)) {
+        const [name, description] = this.schemaUriToNameAndDescription.get(schemaUri);
+        unresolvedJsonSchema.schema.title = name ?? unresolvedJsonSchema.schema.title;
+        unresolvedJsonSchema.schema.description = description ?? unresolvedJsonSchema.schema.description;
+      }
       return unresolvedJsonSchema;
     });
   }
 
-  registerExternalSchema(uri: string, filePatterns?: string[], unresolvedSchema?: JSONSchema): SchemaHandle {
+  registerExternalSchema(
+    uri: string,
+    filePatterns?: string[],
+    unresolvedSchema?: JSONSchema,
+    name?: string,
+    description?: string
+  ): SchemaHandle {
+    if (name || description) {
+      this.schemaUriToNameAndDescription.set(uri, [name, description]);
+    }
     return super.registerExternalSchema(uri, filePatterns, unresolvedSchema);
   }
 

src/languageservice/utils/astUtils.ts

@@ -3,14 +3,18 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { Document, isDocument, isScalar, Node, visit as cstVisit, YAMLMap, YAMLSeq } from 'yaml';
+import { Document, isDocument, isScalar, Node, visit, YAMLMap, YAMLSeq } from 'yaml';
+import { CollectionItem, SourceToken, Token } from 'yaml/dist/parse/cst';
+import { VisitPath } from 'yaml/dist/parse/cst-visit';
+
+type Visitor = (item: SourceToken, path: VisitPath) => number | symbol | Visitor | void;
 
 export function getParent(doc: Document, nodeToFind: Node): Node | undefined {
   let parentNode: Node;
-  cstVisit(doc, (_, node: Node, path) => {
+  visit(doc, (_, node: Node, path) => {
     if (node === nodeToFind) {
       parentNode = path[path.length - 1] as Node;
-      return cstVisit.BREAK;
+      return visit.BREAK;
     }
   });
 
@@ -41,3 +45,74 @@ export function indexOf(seq: YAMLSeq, item: Node): number | undefined {
   }
   return undefined;
 }
+
+/**
+ * Check that given offset is in YAML comment
+ * @param doc the yaml document
+ * @param offset the offset to check
+ */
+export function isInComment(tokens: Token[], offset: number): boolean {
+  let inComment = false;
+  for (const token of tokens) {
+    if (token.type === 'document') {
+      _visit([], (token as unknown) as SourceToken, (item) => {
+        if (isCollectionItem(item) && item.value?.type === 'comment') {
+          if (token.offset <= offset && item.value.source.length + item.value.offset >= offset) {
+            inComment = true;
+            return visit.BREAK;
+          }
+        } else if (item.type === 'comment' && item.offset <= offset && item.offset + item.source.length >= offset) {
+          inComment = true;
+          return visit.BREAK;
+        }
+      });
+    } else if (token.type === 'comment') {
+      if (token.offset <= offset && token.source.length + token.offset >= offset) {
+        return true;
+      }
+    }
+    if (inComment) {
+      break;
+    }
+  }
+
+  return inComment;
+}
+
+function isCollectionItem(token: unknown): token is CollectionItem {
+  return token['start'] !== undefined;
+}
+
+function _visit(path: VisitPath, item: SourceToken, visitor: Visitor): number | symbol | Visitor | void {
+  let ctrl = visitor(item, path);
+  if (typeof ctrl === 'symbol') return ctrl;
+  for (const field of ['key', 'value'] as const) {
+    const token = item[field];
+    if (token && 'items' in token) {
+      for (let i = 0; i < token.items.length; ++i) {
+        const ci = _visit(Object.freeze(path.concat([[field, i]])), token.items[i], visitor);
+        if (typeof ci === 'number') i = ci - 1;
+        else if (ci === visit.BREAK) return visit.BREAK;
+        else if (ci === visit.REMOVE) {
+          token.items.splice(i, 1);
+          i -= 1;
+        }
+      }
+      if (typeof ctrl === 'function' && field === 'key') ctrl = ctrl(item, path);
+    }
+  }
+
+  const token = item['sep'];
+  if (token) {
+    for (let i = 0; i < token.length; ++i) {
+      const ci = _visit(Object.freeze(path), token[i], visitor);
+      if (typeof ci === 'number') i = ci - 1;
+      else if (ci === visit.BREAK) return visit.BREAK;
+      else if (ci === visit.REMOVE) {
+        token.items.splice(i, 1);
+        i -= 1;
+      }
+    }
+  }
+  return typeof ctrl === 'function' ? ctrl(item, path) : ctrl;
+}