feat: DH-19897: Handle missing jsapi/dh-internal.js files on core workers (#2501)

DH-19897: Handle missing jsapi/dh-internal.js files on core workers

I added a flag to resolve 404s as empty content instead of throwing an
error. 404s were already differentiated from other errors that get retry
attempts, so it was a simple change.

Testing
I tested locally by renaming `jsapi/dh-internal.js` in the `loadModules`
call to `jsapi/dh-internal2.js` to mimick a 404. Confirmed empty content
file gets created in VS Code extension.

I also tested stopping the server which produces a different error that
invokes the retry mechanism instead of a 404. Worst case scenario, if we
ever get out of sorts, restarting VS Code will delete the tmp download
folder and retry everything again.

Author: bmingles

packages/jsapi-nodejs/src/errorUtils.ts

@@ -1,3 +1,16 @@
+import { STATUS_CODES } from 'node:http';
+
+/** General Http Error class. */
+export class HttpError extends Error {
+  constructor(
+    public statusCode: number,
+    message?: string
+  ) {
+    super(message ?? STATUS_CODES[statusCode]);
+    this.name = 'HttpError';
+  }
+}
+
 /**
  * Return true if given error has a code:string prop. Optionally check if the
  * code matches a given value.

packages/jsapi-nodejs/src/loaderUtils.ts

@@ -5,22 +5,36 @@ import type { dh as DhType } from '@deephaven/jsapi-types';
 import { downloadFromURL, urlToDirectoryName } from './serverUtils.js';
 import { polyfillWs } from './polyfillWs.js';
 import { ensureDirectoriesExist, getDownloadPaths } from './fsUtils.js';
+import { HttpError } from './errorUtils.js';
 
 type NonEmptyArray<T> = [T, ...T[]];
 
+const DH_CORE_MODULE = 'jsapi/dh-core.js' as const;
+const DH_INTERNAL_MODULE = 'jsapi/dh-internal.js' as const;
+
 /** Transform downloaded content */
 export type PostDownloadTransform = (
   serverPath: string,
   content: string
 ) => string;
 
+export type PostDownloadErrorTransform = (
+  serverPath: string,
+  error: unknown
+) => string;
+
 export type LoadModuleOptions = {
   serverUrl: URL;
   serverPaths: NonEmptyArray<string>;
-  download: boolean | PostDownloadTransform;
   storageDir: string;
   targetModuleType: 'esm' | 'cjs';
-};
+} & (
+  | { download: false }
+  | {
+      download: true | PostDownloadTransform;
+      errorTransform?: PostDownloadErrorTransform;
+    }
+);
 
 /**
  * Load a list of modules from a server.
@@ -32,38 +46,56 @@ export type LoadModuleOptions = {
  * the modules will be downloaded and stored. If set to a `PostDownloadTransform`
  * function, the downloaded content will be passed to the function and the result
  * saved to disk.
+ * @param errorTransform Optional function to transform errors that occur during
+ * the download process. If not provided, errors will be thrown.
  * @param storageDir The directory to store the downloaded modules.
  * @param targetModuleType The type of module to load. Can be either 'esm' or 'cjs'.
  * @returns The default export of the first module in `serverPaths`.
  */
-export async function loadModules<TMainModule>({
-  serverUrl,
-  serverPaths,
-  download,
-  storageDir,
-  targetModuleType,
-}: LoadModuleOptions): Promise<TMainModule> {
+export async function loadModules<TMainModule>(
+  options: LoadModuleOptions
+): Promise<TMainModule> {
   polyfillWs();
 
+  const { serverUrl, serverPaths, storageDir, targetModuleType } = options;
+
   const serverStorageDir = path.join(storageDir, urlToDirectoryName(serverUrl));
 
-  if (download !== false) {
+  if (options.download !== false) {
     ensureDirectoriesExist([serverStorageDir]);
 
+    // Handle rejected Promise from download
+    const handleRejected = (reason: unknown, i: number): string => {
+      if (typeof options.errorTransform === 'function') {
+        return options.errorTransform(serverPaths[i], reason);
+      }
+
+      throw reason;
+    };
+
+    // Handle resolved Promise from download
+    const handleResolved = (value: string, i: number): string => {
+      if (typeof options.download === 'function') {
+        return options.download(serverPaths[i], value);
+      }
+
+      return value;
+    };
+
     // Download from server
     const serverUrls = serverPaths.map(
       serverPath => new URL(serverPath, serverUrl)
     );
-    let contents = await Promise.all(
+
+    const settledResults = await Promise.allSettled(
       serverUrls.map(url => downloadFromURL(url))
     );
 
-    // Post-download transform
-    if (typeof download === 'function') {
-      contents = contents.map((content, i) =>
-        download(serverPaths[i], content)
-      );
-    }
+    const contents: string[] = settledResults.map((result, i) =>
+      result.status === 'rejected'
+        ? handleRejected(result.reason, i)
+        : handleResolved(result.value, i)
+    );
 
     // Write to disk
     const downloadPaths = getDownloadPaths(serverStorageDir, serverPaths);
@@ -111,37 +143,60 @@ export async function loadDhModules({
     globalThis.window = globalThis;
   }
 
+  // If target module type is `cjs`, we need to transform the downloaded content
+  // by replaing some ESM specific syntax with CJS syntax.
+  const cjsDownloadTransform: PostDownloadTransform = (
+    serverPath: string,
+    content: string
+  ): string => {
+    if (serverPath === DH_CORE_MODULE) {
+      return content
+        .replace(
+          `import {dhinternal} from './dh-internal.js';`,
+          `const {dhinternal} = require("./dh-internal.js");`
+        )
+        .replace(`export default dh;`, `module.exports = dh;`);
+    }
+
+    if (serverPath === DH_INTERNAL_MODULE) {
+      return content.replace(
+        `export{__webpack_exports__dhinternal as dhinternal};`,
+        `module.exports={dhinternal:__webpack_exports__dhinternal};`
+      );
+    }
+
+    return content;
+  };
+
+  // `dh-internal.js` module is being removed from future versions of DH core,
+  // but there's not a great way for this library to know whether it's present
+  // or not. Treat 404s as empty content to make things compatible with both
+  // configurations.
+  const errorTransform: PostDownloadErrorTransform = (
+    serverPath: string,
+    error: unknown
+  ): string => {
+    if (
+      serverPath === DH_INTERNAL_MODULE &&
+      error instanceof HttpError &&
+      error.statusCode === 404
+    ) {
+      return '';
+    }
+
+    throw error;
+  };
+
   const coreModule = await loadModules<
     typeof DhType & { default?: typeof DhType }
   >({
     serverUrl,
-    serverPaths: ['jsapi/dh-core.js', 'jsapi/dh-internal.js'],
+    serverPaths: [DH_CORE_MODULE, DH_INTERNAL_MODULE],
     storageDir,
     targetModuleType,
-    download:
-      targetModuleType === 'esm'
-        ? // ESM does not need any transformation since the server modules are already ESM.
-          true
-        : // CJS needs a post-download transform to convert the ESM modules to CJS.
-          (serverPath, content) => {
-            if (serverPath === 'jsapi/dh-core.js') {
-              return content
-                .replace(
-                  `import {dhinternal} from './dh-internal.js';`,
-                  `const {dhinternal} = require("./dh-internal.js");`
-                )
-                .replace(`export default dh;`, `module.exports = dh;`);
-            }
-
-            if (serverPath === 'jsapi/dh-internal.js') {
-              return content.replace(
-                `export{__webpack_exports__dhinternal as dhinternal};`,
-                `module.exports={dhinternal:__webpack_exports__dhinternal};`
-              );
-            }
-
-            return content;
-          },
+    // Download the module and transform it if the target module type is `cjs`.
+    download: targetModuleType === 'esm' ? true : cjsDownloadTransform,
+    errorTransform,
   });
 
   // ESM uses `default` export. CJS does not.

packages/jsapi-nodejs/src/serverUtils.ts

@@ -1,6 +1,6 @@
 import * as http from 'node:http';
 import * as https from 'node:https';
-import { hasErrorCode, isAggregateError } from './errorUtils.js';
+import { hasErrorCode, HttpError, isAggregateError } from './errorUtils.js';
 
 export const SERVER_STATUS_CHECK_TIMEOUT = 3000;
 
@@ -46,7 +46,7 @@ export async function downloadFromURL(
 
         res.on('end', async () => {
           if (res.statusCode === 404) {
-            reject(new Error(`File not found: "${url}"`));
+            reject(new HttpError(404, `File not found: "${url}"`));
             return;
           }