feat: auto-detect Kubernetes crd schema (#1050)

* feat: auto-detect Kubernetes schema

automatically detect the Kubernetes schema based on the document's
GroupVersionKind (GVK) and retrieve the matching schema from the
CRD catalog.

* fix: only auto-detect k8s crd for k8s files

* feat: check if GVK in main kubeSchema

* fix: test

* fix: promises were made

* fix: rename crd options and document them

* fix eslint errors

* fix tests

* fix: do not use CRDs-catalog for builtin k8s ressources

* fix: clarify @ts-ignore

* fix: import type from ../jsonSchema

* Handle case where custom schema provider is present

The CRD schema resolving logic now runs even if a custom schema provider
is present,
allowing this to work properly in vscode-yaml.

Signed-off-by: David Thompson <davthomp@redhat.com>

* Add logic to handle OpenShift CRDs

The CRD schemas for OpenShift are layed out slightly different from
everything else in the CRD schema repo in order to provide different
sets of schemas for different versions of OpenShift.

Signed-off-by: David Thompson <davthomp@redhat.com>

---------

Signed-off-by: David Thompson <davthomp@redhat.com>
Co-authored-by: Marius Svechla <m.svechla@gmail.com>
Co-authored-by: Roland Wahl <rwahl@protonmail.com>
Co-authored-by: David Thompson <davthomp@redhat.com>

Author: 4 people

README.md

@@ -41,6 +41,8 @@ The following settings are supported:
 - `yaml.schemas`: Helps you associate schemas with files in a glob pattern
 - `yaml.schemaStore.enable`: When set to true the YAML language server will pull in all available schemas from [JSON Schema Store](https://www.schemastore.org)
 - `yaml.schemaStore.url`: URL of a schema store catalog to use when downloading schemas.
+- `yaml.kubernetesCRDStore.enable`: When set to true the YAML language server will parse Kubernetes CRDs automatically and download them from the [CRD store](https://github.com/datreeio/CRDs-catalog).
+- `yaml.kubernetesCRDStore.url`: URL of a crd store catalog to use when downloading schemas. Defaults to `https://raw.githubusercontent.com/datreeio/CRDs-catalog/main`.
 - `yaml.customTags`: Array of custom tags that the parser will validate against. It has two ways to be used. Either an item in the array is a custom tag such as "!Ref" and it will automatically map !Ref to scalar or you can specify the type of the object !Ref should be e.g. "!Ref sequence". The type of object can be either scalar (for strings and booleans), sequence (for arrays), map (for objects).
 - `yaml.maxItemsComputed`: The maximum number of outline symbols and folding regions computed (limited for performance reasons).
 - `[yaml].editor.tabSize`: the number of spaces to use when autocompleting. Takes priority over editor.tabSize.

src/languageserver/handlers/settingsHandlers.ts

@@ -84,6 +84,14 @@ export class SettingsHandler {
           this.yamlSettings.schemaStoreUrl = settings.yaml.schemaStore.url;
         }
       }
+
+      if (settings.yaml.kubernetesCRDStore) {
+        this.yamlSettings.kubernetesCRDStoreEnabled = settings.yaml.kubernetesCRDStore.enable;
+        if (settings.yaml.kubernetesCRDStore.url?.length !== 0) {
+          this.yamlSettings.kubernetesCRDStoreUrl = settings.yaml.kubernetesCRDStore.url;
+        }
+      }
+
       if (settings.files?.associations) {
         for (const [ext, languageId] of Object.entries(settings.files.associations)) {
           if (languageId === 'yaml') {

src/languageservice/services/crdUtil.ts

@@ -0,0 +1,92 @@
+import { SingleYAMLDocument } from '../parser/yamlParser07';
+import { JSONDocument } from '../parser/jsonParser07';
+
+import { ResolvedSchema } from 'vscode-json-languageservice/lib/umd/services/jsonSchemaService';
+import { JSONSchema } from '../jsonSchema';
+
+/**
+ * Retrieve schema by auto-detecting the Kubernetes GroupVersionKind (GVK) from the document.
+ * If there is no definition for the GVK in the main kubernetes schema,
+ * the schema is then retrieved from the CRD catalog.
+ * Public for testing purpose, not part of the API.
+ * @param doc
+ * @param crdCatalogURI The URL of the CRD catalog to retrieve the schema from
+ * @param kubernetesSchema The main kubernetes schema, if it includes a definition for the GVK it will be used
+ */
+export function autoDetectKubernetesSchemaFromDocument(
+  doc: SingleYAMLDocument | JSONDocument,
+  crdCatalogURI: string,
+  kubernetesSchema: ResolvedSchema
+): string | undefined {
+  const res = getGroupVersionKindFromDocument(doc);
+  if (!res) {
+    return undefined;
+  }
+  const { group, version, kind } = res;
+  if (!group || !version || !kind) {
+    return undefined;
+  }
+
+  const k8sSchema: JSONSchema = kubernetesSchema.schema;
+  const kubernetesBuildIns: string[] = (k8sSchema.oneOf || [])
+    .map((s) => {
+      if (typeof s === 'boolean') {
+        return undefined;
+      }
+      return s._$ref || s.$ref;
+    })
+    .filter((ref) => ref)
+    .map((ref) => ref.replace('_definitions.json#/definitions/', '').toLowerCase());
+  const groupWithoutK8sIO = group.replace('.k8s.io', '');
+  const k8sTypeName = `io.k8s.api.${groupWithoutK8sIO.toLowerCase()}.${version.toLowerCase()}.${kind.toLowerCase()}`;
+
+  if (kubernetesBuildIns.includes(k8sTypeName)) {
+    return undefined;
+  }
+
+  if (k8sTypeName.includes('openshift.io')) {
+    return `${crdCatalogURI}/openshift/v4.15-strict/${kind.toLowerCase()}_${group.toLowerCase()}_${version.toLowerCase()}.json`;
+  }
+
+  const schemaURL = `${crdCatalogURI}/${group.toLowerCase()}/${kind.toLowerCase()}_${version.toLowerCase()}.json`;
+  return schemaURL;
+}
+
+/**
+ * Retrieve the group, version and kind from the document.
+ * Public for testing purpose, not part of the API.
+ * @param doc
+ */
+export function getGroupVersionKindFromDocument(
+  doc: SingleYAMLDocument | JSONDocument
+): { group: string; version: string; kind: string } | undefined {
+  if (doc instanceof SingleYAMLDocument) {
+    try {
+      const rootJSON = doc.root.internalNode.toJSON();
+      if (!rootJSON) {
+        return undefined;
+      }
+
+      const groupVersion = rootJSON['apiVersion'];
+      if (!groupVersion) {
+        return undefined;
+      }
+
+      const [group, version] = groupVersion.split('/');
+      if (!group || !version) {
+        return undefined;
+      }
+
+      const kind = rootJSON['kind'];
+      if (!kind) {
+        return undefined;
+      }
+
+      return { group, version, kind };
+    } catch (error) {
+      console.error('Error parsing YAML document:', error);
+      return undefined;
+    }
+  }
+  return undefined;
+}

src/languageservice/services/yamlSchemaService.ts

@@ -6,6 +6,7 @@
 
 import { JSONSchema, JSONSchemaMap, JSONSchemaRef } from '../jsonSchema';
 import { SchemaPriority, SchemaRequestService, WorkspaceContextService } from '../yamlLanguageService';
+import { SettingsState } from '../../yamlSettings';
 import {
   UnresolvedSchema,
   ResolvedSchema,
@@ -31,6 +32,8 @@ import Ajv, { DefinedError, type AnySchemaObject, type ValidateFunction } from '
 import Ajv4 from 'ajv-draft-04';
 import Ajv2019 from 'ajv/dist/2019';
 import Ajv2020 from 'ajv/dist/2020';
+import { autoDetectKubernetesSchemaFromDocument } from './crdUtil';
+import { CRD_CATALOG_URL, KUBERNETES_SCHEMA_URL } from '../utils/schemaUrls';
 
 const ajv4 = new Ajv4({ allErrors: true });
 const ajv7 = new Ajv({ allErrors: true });
@@ -120,19 +123,22 @@ export class YAMLSchemaService extends JSONSchemaService {
   private filePatternAssociations: JSONSchemaService.FilePatternAssociation[];
   private contextService: WorkspaceContextService;
   private requestService: SchemaRequestService;
+  private yamlSettings: SettingsState;
   public schemaPriorityMapping: Map<string, Set<SchemaPriority>>;
 
   private schemaUriToNameAndDescription = new Map<string, SchemaStoreSchema>();
 
   constructor(
     requestService: SchemaRequestService,
     contextService?: WorkspaceContextService,
-    promiseConstructor?: PromiseConstructor
+    promiseConstructor?: PromiseConstructor,
+    yamlSettings?: SettingsState
   ) {
     super(requestService, contextService, promiseConstructor);
     this.customSchemaProvider = undefined;
     this.requestService = requestService;
     this.schemaPriorityMapping = new Map();
+    this.yamlSettings = yamlSettings;
   }
 
   registerCustomSchemaProvider(customSchemaProvider: CustomSchemaProvider): void {
@@ -408,16 +414,35 @@ export class YAMLSchemaService extends JSONSchemaService {
     };
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const resolveSchema = (): any => {
+    const resolveSchema = async (): Promise<any> => {
       const seen: { [schemaId: string]: boolean } = Object.create(null);
       const schemas: string[] = [];
+      let k8sAllSchema: ResolvedSchema = undefined;
 
       for (const entry of this.filePatternAssociations) {
         if (entry.matchesPattern(resource)) {
           for (const schemaId of entry.getURIs()) {
             if (!seen[schemaId]) {
-              schemas.push(schemaId);
-              seen[schemaId] = true;
+              if (this.yamlSettings?.kubernetesCRDStoreEnabled && schemaId === KUBERNETES_SCHEMA_URL) {
+                if (!k8sAllSchema) {
+                  k8sAllSchema = await this.getResolvedSchema(KUBERNETES_SCHEMA_URL);
+                }
+                const kubeSchema = autoDetectKubernetesSchemaFromDocument(
+                  doc,
+                  this.yamlSettings.kubernetesCRDStoreUrl ?? CRD_CATALOG_URL,
+                  k8sAllSchema
+                );
+                if (kubeSchema) {
+                  schemas.push(kubeSchema);
+                  seen[schemaId] = true;
+                } else {
+                  schemas.push(schemaId);
+                  seen[schemaId] = true;
+                }
+              } else {
+                schemas.push(schemaId);
+                seen[schemaId] = true;
+              }
             }
           }
         }
@@ -435,6 +460,7 @@ export class YAMLSchemaService extends JSONSchemaService {
     if (modelineSchema) {
       return resolveSchemaForResource([modelineSchema]);
     }
+
     if (this.customSchemaProvider) {
       return this.customSchemaProvider(resource)
         .then((schemaUri) => {
@@ -477,9 +503,8 @@ export class YAMLSchemaService extends JSONSchemaService {
             return resolveSchema();
           }
         );
-    } else {
-      return resolveSchema();
     }
+    return resolveSchema();
   }
 
   // Set the priority of a schema in the schema service

src/languageservice/utils/schemaUrls.ts

@@ -8,6 +8,7 @@ import { isRelativePath, relativeToAbsolutePath } from './paths';
 export const KUBERNETES_SCHEMA_URL =
   'https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.32.1-standalone-strict/all.json';
 export const JSON_SCHEMASTORE_URL = 'https://www.schemastore.org/api/json/catalog.json';
+export const CRD_CATALOG_URL = 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main';
 
 export function checkSchemaURI(
   workspaceFolders: WorkspaceFolder[],

src/languageservice/yamlLanguageService.ts

@@ -197,7 +197,7 @@ export function getLanguageService(params: {
   yamlSettings?: SettingsState;
   clientCapabilities?: ClientCapabilities;
 }): LanguageService {
-  const schemaService = new YAMLSchemaService(params.schemaRequestService, params.workspaceContext);
+  const schemaService = new YAMLSchemaService(params.schemaRequestService, params.workspaceContext, null, params.yamlSettings);
   const completer = new YamlCompletion(schemaService, params.clientCapabilities, yamlDocumentsCache, params.telemetry);
   const hover = new YAMLHover(schemaService, params.telemetry);
   const yamlDocumentSymbols = new YAMLDocumentSymbols(schemaService, params.telemetry);

src/yamlSettings.ts

@@ -4,7 +4,7 @@ import { ISchemaAssociations } from './requestTypes';
 import { URI } from 'vscode-uri';
 import { JSONSchema } from './languageservice/jsonSchema';
 import { TextDocument } from 'vscode-languageserver-textdocument';
-import { JSON_SCHEMASTORE_URL } from './languageservice/utils/schemaUrls';
+import { CRD_CATALOG_URL, JSON_SCHEMASTORE_URL } from './languageservice/utils/schemaUrls';
 import { YamlVersion } from './languageservice/parser/yamlParser07';
 
 // Client settings interface to grab settings relevant for the language server
@@ -20,6 +20,10 @@ export interface Settings {
       url: string;
       enable: boolean;
     };
+    kubernetesCRDStore: {
+      url: string;
+      enable: boolean;
+    };
     disableDefaultProperties: boolean;
     disableAdditionalProperties: boolean;
     suggest: {
@@ -78,6 +82,8 @@ export class SettingsState {
   customTags = [];
   schemaStoreEnabled = true;
   schemaStoreUrl = JSON_SCHEMASTORE_URL;
+  kubernetesCRDStoreEnabled = true;
+  kubernetesCRDStoreUrl = CRD_CATALOG_URL;
   indentation: string | undefined = undefined;
   disableAdditionalProperties = false;
   disableDefaultProperties = false;

test/schema.test.ts

@@ -14,6 +14,7 @@ import { LanguageService, SchemaPriority } from '../src';
 import { MarkupContent, Position } from 'vscode-languageserver-types';
 import { LineCounter } from 'yaml';
 import { getSchemaFromModeline } from '../src/languageservice/services/modelineUtil';
+import { getGroupVersionKindFromDocument } from '../src/languageservice/services/crdUtil';
 
 const requestServiceMock = function (uri: string): Promise<string> {
   return Promise.reject<string>(`Resource ${uri} not found.`);
@@ -701,6 +702,70 @@ describe('JSON Schema', () => {
     });
   });
 
+  describe('Test getGroupVersionKindFromDocument', function () {
+    it('builtin kubernetes resource group should not get resolved', async () => {
+      checkReturnGroupVersionKind('apiVersion: v1\nkind: Pod', true, undefined, 'v1', 'Pod');
+    });
+
+    it('builtin kubernetes resource with complex apiVersion should get resolved ', async () => {
+      checkReturnGroupVersionKind(
+        'apiVersion: admissionregistration.k8s.io/v1\nkind: MutatingWebhook',
+        false,
+        'admissionregistration.k8s.io',
+        'v1',
+        'MutatingWebhook'
+      );
+    });
+
+    it('custom argo application CRD should get resolved', async () => {
+      checkReturnGroupVersionKind(
+        'apiVersion: argoproj.io/v1alpha1\nkind: Application',
+        false,
+        'argoproj.io',
+        'v1alpha1',
+        'Application'
+      );
+    });
+
+    it('custom argo application CRD with whitespace should get resolved', async () => {
+      checkReturnGroupVersionKind(
+        'apiVersion: argoproj.io/v1alpha1\nkind: Application ',
+        false,
+        'argoproj.io',
+        'v1alpha1',
+        'Application'
+      );
+    });
+
+    it('custom argo application CRD with other fields should get resolved', async () => {
+      checkReturnGroupVersionKind(
+        'someOtherVal: test\napiVersion: argoproj.io/v1alpha1\nkind: Application\nmetadata:\n  name: my-app',
+        false,
+        'argoproj.io',
+        'v1alpha1',
+        'Application'
+      );
+    });
+
+    function checkReturnGroupVersionKind(
+      content: string,
+      error: boolean,
+      expectedGroup: string,
+      expectedVersion: string,
+      expectedKind: string
+    ): void {
+      const yamlDoc = parser.parse(content);
+      const res = getGroupVersionKindFromDocument(yamlDoc.documents[0]);
+      if (error) {
+        assert.strictEqual(res, undefined);
+      } else {
+        assert.strictEqual(res.group, expectedGroup);
+        assert.strictEqual(res.version, expectedVersion);
+        assert.strictEqual(res.kind, expectedKind);
+      }
+    }
+  });
+
   describe('Test getSchemaFromModeline', function () {
     it('simple case', async () => {
       checkReturnSchemaUrl('# yaml-language-server: $schema=expectedUrl', 'expectedUrl');