Merge pull request redhat-developer#1 from gabemontero/rhdh-rhoai-catalogprocessor-prototype

RHDHPAI-506: custom catalog process with new location type for rhdh/rhoai 'bridge'

Author: gabemontero

workspaces/rhdh-ai/app-config.yaml

@@ -12,6 +12,12 @@ backend:
   # auth:
   #   keys:
   #     - secret: ${BACKEND_SECRET}
+  auth:
+    externalAccess:
+    - type: static
+      options:
+        token: '${RHDH_TOKEN}'
+        subject: admin-curl-access
   baseUrl: http://localhost:7007
   listen:
     port: 7007
@@ -31,6 +37,11 @@ backend:
     client: better-sqlite3
     connection: ':memory:'
   # workingDirectory: /tmp # Use this to configure a working directory for the scaffolder, defaults to the OS temp-dir
+  reading:
+    allow:
+      - host: example.com
+      - host: '*.mozilla.org'
+      - host: '${RHDH_BRIDGE_HOST}'
 
 integrations:
   github:

workspaces/rhdh-ai/packages/backend/src/index.ts

@@ -15,6 +15,10 @@
  */
 
 import { createBackend } from '@backstage/backend-defaults';
+import {
+  catalogModuleRHDHRHOAIReaderProcessor,
+  catalogModuleRHDHRHOAILocationsExtensionPoint,
+} from '@red-hat-developer-hub/backstage-plugin-catalog-backend-module-rhdh-ai';
 
 const backend = createBackend();
 
@@ -63,4 +67,6 @@ backend.add(
     '@red-hat-developer-hub/backstage-plugin-catalog-backend-module-rhdh-ai'
   ),
 );
+backend.add(catalogModuleRHDHRHOAILocationsExtensionPoint);
+backend.add(catalogModuleRHDHRHOAIReaderProcessor);
 backend.start();

workspaces/rhdh-ai/plugins/catalog-backend-module-rhdh-ai/package.json

@@ -27,6 +27,7 @@
     "@backstage/backend-plugin-api": "^1.1.1",
     "@backstage/catalog-model": "^1.7.2",
     "@backstage/errors": "^1.2.7",
+    "@backstage/plugin-catalog-common": "^1.1.3",
     "@backstage/plugin-catalog-node": "^1.15.1"
   },
   "devDependencies": {

workspaces/rhdh-ai/plugins/catalog-backend-module-rhdh-ai/src/index.ts

@@ -22,5 +22,8 @@
 
 // export * from './clients';
 export { catalogModuleModelCatalogResourceEntityProvider as default } from './module';
+export { catalogModuleRHDHRHOAIReaderProcessor } from './module';
+export { catalogModuleRHDHRHOAILocationsExtensionPoint } from './module';
 export * from './providers';
+export * from './processors';
 export * from './clients';

workspaces/rhdh-ai/plugins/catalog-backend-module-rhdh-ai/src/module.ts

@@ -17,9 +17,13 @@ import {
   coreServices,
   createBackendModule,
 } from '@backstage/backend-plugin-api';
-import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import {
+  catalogProcessingExtensionPoint,
+  catalogLocationsExtensionPoint,
+} from '@backstage/plugin-catalog-node/alpha';
 
 import { ModelCatalogResourceEntityProvider } from './providers';
+import { RHDHRHOAIReaderProcessor } from './processors';
 
 export const catalogModuleModelCatalogResourceEntityProvider =
   createBackendModule({
@@ -50,3 +54,42 @@ export const catalogModuleModelCatalogResourceEntityProvider =
       });
     },
   });
+
+export const catalogModuleRHDHRHOAIReaderProcessor = createBackendModule({
+  pluginId: 'catalog',
+  moduleId: 'rhdh-rhoai-bridge-reader-processor',
+  register(env) {
+    env.registerInit({
+      deps: {
+        catalog: catalogProcessingExtensionPoint,
+        reader: coreServices.urlReader,
+      },
+      async init({ catalog, reader }) {
+        catalog.addProcessor(new RHDHRHOAIReaderProcessor(reader));
+      },
+    });
+  },
+});
+
+// so a `CatalogProcessor` does not need to also provide a `CatalogLocationsExtensionPoint` if it only supports imports of locations
+// from the app-config.yaml on startup, but if you want to dynamically add Locations via the catalog's REST API (like what the UI does for import
+// of catalog-info.yaml from git repos) then you need to also provide a `CatalogLocationsExtension` point to add your type to the default list of 'url' and 'file';
+// fwiw in examining the core Backstage code, none of the default `CatalogProcessors` bother to also provide a CatalogLocationsExtension`; however,
+// we want to allow our RHDH bridge to import new locations dynamically
+export const catalogModuleRHDHRHOAILocationsExtensionPoint =
+  createBackendModule({
+    pluginId: 'catalog',
+    moduleId: 'rhdh-rhoai-bridge-location-extension-point',
+    register(env) {
+      env.registerInit({
+        deps: {
+          catalog: catalogLocationsExtensionPoint,
+        },
+        async init({ catalog }) {
+          // setAllowedLocationTypes does not add to the list but replaces it, so we preserve the default options of 'file' and 'url'
+          const allowedLocationTypes = ['file', 'url', 'rhdh-rhoai-bridge'];
+          catalog.setAllowedLocationTypes(allowedLocationTypes);
+        },
+      });
+    },
+  });

workspaces/rhdh-ai/plugins/catalog-backend-module-rhdh-ai/src/processors/RHDHRHOAIReaderProcessor.ts

@@ -0,0 +1,93 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  processingResult,
+  CatalogProcessor,
+  CatalogProcessorEmit,
+  CatalogProcessorParser,
+  CatalogProcessorResult,
+} from '@backstage/plugin-catalog-node';
+import { UrlReaderService } from '@backstage/backend-plugin-api';
+
+import { LocationSpec } from '@backstage/plugin-catalog-common';
+
+// A processor that reads from the RHDH RHOAI Bridge
+export class RHDHRHOAIReaderProcessor implements CatalogProcessor {
+  constructor(private readonly reader: UrlReaderService) {}
+
+  getProcessorName(): string {
+    return 'RHDHRHOAIReaderProcessor';
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+    parser: CatalogProcessorParser,
+  ): Promise<boolean> {
+    // Pick a custom location type string. A location will be
+    // registered later with this type.
+    if (location.type !== 'rhdh-rhoai-bridge') {
+      return false;
+    }
+
+    try {
+      // Use the builtin reader facility to grab data from the
+      // API. If you prefer, you can just use plain fetch here
+      // (from the node-fetch package), or any other method of
+      // your choosing.
+      // TODO eventually we will want to take the k8s credentials provided to
+      // backstage and its k8s plugin and supply them to the bridge
+      // for potential auth and access control checks (i.e. SARs) with the kubeflow MR
+      const data = await this.reader.readUrl(location.target);
+      const response = [{ url: location.target, data: await data.buffer() }];
+      // Repeatedly call emit(processingResult.entity(location, <entity>))
+      const parseResults: CatalogProcessorResult[] = [];
+      for (const item of response) {
+        for await (const parseResult of parser({
+          data: item.data,
+          location: { type: location.type, target: item.url },
+        })) {
+          parseResults.push(parseResult);
+          emit(parseResult);
+        }
+      }
+      emit(processingResult.refresh(`${location.type}:${location.target}`));
+    } catch (error) {
+      const message = `Unable to read ${location.type}, ${error}`.substring(
+        0,
+        5000,
+      );
+      // TODO when we enable cache this is how UrlReaderPRocessor.ts in core backstage handles errors
+      // if (error.name === 'NotModifiedError' && cacheItem) {
+      //   for (const parseResult of cacheItem.value) {
+      //     emit(parseResult);
+      //   }
+      //   emit(processingResult.refresh(`${location.type}:${location.target}`));
+      //   await cache.set(CACHE_KEY, cacheItem);
+      // } else if (error.name === 'NotFoundError') {
+      //   if (!optional) {
+      //     emit(processingResult.notFoundError(location, message));
+      //   }
+      // } else {
+      //   emit(processingResult.generalError(location, message));
+      // }
+      emit(processingResult.generalError(location, message));
+    }
+
+    return true;
+  }
+}

workspaces/rhdh-ai/plugins/catalog-backend-module-rhdh-ai/src/processors/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { RHDHRHOAIReaderProcessor } from './RHDHRHOAIReaderProcessor';

workspaces/rhdh-ai/yarn.lock

@@ -10175,6 +10175,7 @@ __metadata:
     "@backstage/catalog-model": ^1.7.2
     "@backstage/cli": ^0.29.5
     "@backstage/errors": ^1.2.7
+    "@backstage/plugin-catalog-common": ^1.1.3
     "@backstage/plugin-catalog-node": ^1.15.1
   languageName: unknown
   linkType: soft