fix(core): improve plugin worker error messages and lifecycle timeouts (#34251)

## Current Behavior
Plugin workers occasionally fall over during the start up steps. 

## Expected Behavior
Improves some issues with the error handling when loading plugin workers
and adds some more logs to help understand what's went wrong here.

## Related Issue(s)
<!-- Please link the issue being fixed so it gets closed when this is
merged. -->

Fixes #

---------

Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: nx-cloud[bot] <71083854+nx-cloud[bot]@users.noreply.github.com>
Co-authored-by: AgentEnder <AgentEnder@users.noreply.github.com>

Author: 3 people

e2e/release/src/independent-projects.test.ts

@@ -237,7 +237,11 @@ describe('nx release - independent projects', () => {
       const versionWithGitActionsCLIOutput = runCLI(
         `release version 999.9.9-version-git-operations-test.2 -p ${pkg1} --git-commit --git-tag --verbose` // add verbose so we get richer output
       );
-      expect(versionWithGitActionsCLIOutput).toMatchInlineSnapshot(`
+      const filteredOutput = versionWithGitActionsCLIOutput.replace(
+        /\[plugin-(pool|worker)\].*\n/g,
+        ''
+      );
+      expect(filteredOutput).toMatchInlineSnapshot(`
 
         NX   Your filter "{project-name}" matched the following projects:
 
@@ -320,7 +324,11 @@ describe('nx release - independent projects', () => {
       const versionWithGitActionsConfigOutput = runCLI(
         `release version 999.9.9-version-git-operations-test.3 --verbose --gitTag` // add verbose so we get richer output
       );
-      expect(versionWithGitActionsConfigOutput).toMatchInlineSnapshot(`
+      const filteredConfigOutput = versionWithGitActionsConfigOutput.replace(
+        /\[plugin-(pool|worker)\].*\n/g,
+        ''
+      );
+      expect(filteredConfigOutput).toMatchInlineSnapshot(`
 
         NX   Running release version for project: {project-name}
 
@@ -516,7 +524,11 @@ describe('nx release - independent projects', () => {
       const versionWithGitActionsCLIOutput = runCLI(
         `release changelog 999.9.9-changelog-git-operations-test.1 -p ${pkg1} --verbose`
       );
-      expect(versionWithGitActionsCLIOutput).toMatchInlineSnapshot(`
+      const filteredChangelogOutput = versionWithGitActionsCLIOutput.replace(
+        /\[plugin-(pool|worker)\].*\n/g,
+        ''
+      );
+      expect(filteredChangelogOutput).toMatchInlineSnapshot(`
 
         NX   Your filter "{project-name}" matched the following projects:
 

e2e/release/src/independent-projects.workspaces.test.ts

@@ -54,6 +54,8 @@ expect.addSnapshotSerializer({
         .replaceAll('pnpm install --lockfile-only', '{lock-file-command}')
         .replaceAll(getSelectedPackageManager(), '{package-manager}')
         .replaceAll(e2eRegistryUrl, '{registryUrl}')
+        // Filter out plugin worker verbose logs
+        .replaceAll(/\[plugin-(pool|worker)\].*\n/g, '')
         // We trim each line to reduce the chances of snapshot flakiness
         .split('\n')
         .map((r) => r.trim())

e2e/release/src/preserve-local-dependency-protocols.test.ts

@@ -47,6 +47,8 @@ expect.addSnapshotSerializer({
           /Integrity:\s*.*/g,
           'Integrity: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
         )
+        // Filter out plugin worker verbose logs
+        .replaceAll(/\[plugin-(pool|worker)\].*\n/g, '')
 
         .split('\n')
         .map((r) => r.trim())

e2e/release/src/version-plans-check.test.ts

@@ -32,6 +32,8 @@ expect.addSnapshotSerializer({
         .replaceAll(/Test @[\w\d]+/g, 'Test @{COMMIT_AUTHOR}')
         // Normalize the version title date.
         .replaceAll(/\(\d{4}-\d{2}-\d{2}\)/g, '(YYYY-MM-DD)')
+        // Filter out plugin worker verbose logs
+        .replaceAll(/\[plugin-(pool|worker)\].*\n/g, '')
         // We trim each line to reduce the chances of snapshot flakiness
         .split('\n')
         .map((r) => r.trim())

e2e/release/src/version-plans-only-touched.test.ts

@@ -29,6 +29,8 @@ expect.addSnapshotSerializer({
         .replaceAll(/Test @[\w\d]+/g, 'Test @{COMMIT_AUTHOR}')
         // Normalize the version title date.
         .replaceAll(/\(\d{4}-\d{2}-\d{2}\)/g, '(YYYY-MM-DD)')
+        // Filter out plugin worker verbose logs
+        .replaceAll(/\[plugin-(pool|worker)\].*\n/g, '')
         // We trim each line to reduce the chances of snapshot flakiness
         .split('\n')
         .map((r) => r.trim())

packages/nx/src/project-graph/plugins/get-plugins.ts

@@ -47,14 +47,35 @@ export async function getPlugins(
   }
 
   currentPluginsConfigurationHash = pluginsConfigurationHash;
-  const [defaultPlugins, specifiedPlugins] = await Promise.all([
+  const results = await Promise.allSettled([
     getOnlyDefaultPlugins(root),
     (pendingPluginsPromise ??= loadSpecifiedNxPlugins(
       pluginsConfiguration,
       root
     )),
   ]);
 
+  const errors: Error[] = [];
+  const defaultPlugins: LoadedNxPlugin[] = [];
+  const specifiedPlugins: LoadedNxPlugin[] = [];
+
+  for (let i = 0; i < results.length; i++) {
+    const result = results[i];
+    if (result.status === 'fulfilled') {
+      (i === 0 ? defaultPlugins : specifiedPlugins).push(...result.value);
+    } else {
+      errors.push(
+        result.reason instanceof Error
+          ? result.reason
+          : new Error(String(result.reason))
+      );
+    }
+  }
+
+  if (errors.length > 0) {
+    throw new AggregateError(errors, errors.map((e) => e.message).join('\n'));
+  }
+
   loadedPlugins = specifiedPlugins.concat(defaultPlugins);
 
   return loadedPlugins;
@@ -115,28 +136,58 @@ async function loadDefaultNxPlugins(root = workspaceRoot) {
   const plugins = getDefaultPlugins(root);
 
   const cleanupFunctions: Array<() => void> = [];
+  const results = await Promise.allSettled(
+    plugins.map(async (plugin) => {
+      performance.mark(`Load Nx Plugin: ${plugin} - start`);
+
+      const [loadedPluginPromise, cleanup] = await loadingMethod(plugin, root);
+
+      cleanupFunctions.push(cleanup);
+      const res = await loadedPluginPromise;
+      performance.mark(`Load Nx Plugin: ${plugin} - end`);
+      performance.measure(
+        `Load Nx Plugin: ${plugin}`,
+        `Load Nx Plugin: ${plugin} - start`,
+        `Load Nx Plugin: ${plugin} - end`
+      );
+
+      return res;
+    })
+  );
+
+  const defaultPluginResults: LoadedNxPlugin[] = [];
+  const errors: Array<{ pluginName: string; error: Error }> = [];
+
+  for (let i = 0; i < results.length; i++) {
+    const result = results[i];
+    if (result.status === 'fulfilled') {
+      defaultPluginResults.push(result.value);
+    } else {
+      errors.push({
+        pluginName: plugins[i],
+        error:
+          result.reason instanceof Error
+            ? result.reason
+            : new Error(String(result.reason)),
+      });
+    }
+  }
+
+  if (errors.length > 0) {
+    for (const fn of cleanupFunctions) {
+      fn();
+    }
+    const errorMessage = errors
+      .map((e) => `  - ${e.pluginName}: ${e.error.message}`)
+      .join('\n');
+    throw new AggregateError(
+      errors.map((e) => e.error),
+      `Failed to load ${errors.length} default Nx plugin(s):\n${errorMessage}`
+    );
+  }
+
   const ret = [
-    await Promise.all(
-      plugins.map(async (plugin) => {
-        performance.mark(`Load Nx Plugin: ${plugin} - start`);
-
-        const [loadedPluginPromise, cleanup] = await loadingMethod(
-          plugin,
-          root
-        );
-
-        cleanupFunctions.push(cleanup);
-        const res = await loadedPluginPromise;
-        performance.mark(`Load Nx Plugin: ${plugin} - end`);
-        performance.measure(
-          `Load Nx Plugin: ${plugin}`,
-          `Load Nx Plugin: ${plugin} - start`,
-          `Load Nx Plugin: ${plugin} - end`
-        );
-
-        return res;
-      })
-    ),
+    defaultPluginResults,
     () => {
       for (const fn of cleanupFunctions) {
         fn();
@@ -170,7 +221,7 @@ async function loadSpecifiedNxPlugins(
   pluginsConfigurations ??= [];
 
   const cleanupFunctions: Array<() => void> = [];
-  const plugins = await Promise.all(
+  const results = await Promise.allSettled(
     pluginsConfigurations.map(async (plugin, index) => {
       const pluginPath = typeof plugin === 'string' ? plugin : plugin.plugin;
       performance.mark(`Load Nx Plugin: ${pluginPath} - start`);
@@ -201,6 +252,40 @@ async function loadSpecifiedNxPlugins(
     'loadSpecifiedNxPlugins:end'
   );
 
+  const plugins: LoadedNxPlugin[] = [];
+  const errors: Array<{ pluginName: string; error: Error }> = [];
+
+  for (let i = 0; i < results.length; i++) {
+    const result = results[i];
+    if (result.status === 'fulfilled') {
+      plugins.push(result.value);
+    } else {
+      const pluginConfig = pluginsConfigurations[i];
+      const pluginName =
+        typeof pluginConfig === 'string' ? pluginConfig : pluginConfig.plugin;
+      errors.push({
+        pluginName,
+        error:
+          result.reason instanceof Error
+            ? result.reason
+            : new Error(String(result.reason)),
+      });
+    }
+  }
+
+  if (errors.length > 0) {
+    for (const fn of cleanupFunctions) {
+      fn();
+    }
+    const errorMessage = errors
+      .map((e) => `  - ${e.pluginName}: ${e.error.message}`)
+      .join('\n');
+    throw new AggregateError(
+      errors.map((e) => e.error),
+      `Failed to load ${errors.length} Nx plugin(s):\n${errorMessage}`
+    );
+  }
+
   cleanupSpecifiedPlugins = () => {
     for (const fn of cleanupFunctions) {
       fn();

packages/nx/src/project-graph/plugins/isolation/plugin-pool.ts

@@ -4,8 +4,7 @@ import path = require('path');
 
 import { PluginConfiguration } from '../../../config/nx-json';
 
-// TODO (@AgentEnder): After scoped verbose logging is implemented, re-add verbose logs here.
-// import { logger } from '../../utils/logger';
+import { logger } from '../../../utils/logger';
 
 import { getPluginOsSocketPath } from '../../../daemon/socket-utils';
 import { consumeMessagesFromSocket } from '../../../utils/consume-messages-from-socket';
@@ -189,6 +188,9 @@ function createWorkerHandler(
           const { name, createNodesPattern, include, exclude } = result;
           pluginName = name;
           pluginNames.set(worker, pluginName);
+          logger.verbose(
+            `[plugin-pool] loaded plugin "${name}" from worker (pid: ${worker.pid})`
+          );
           onload({
             name,
             include,
@@ -431,6 +433,7 @@ function registerPendingPromise(
 global.nxPluginWorkerCount ??= 0;
 
 async function startPluginWorker(name: string) {
+  performance.mark(`start-plugin-worker:${name}`);
   // this should only really be true when running unit tests within
   // the Nx repo. We still need to start the worker in this case,
   // but its typescript.
@@ -471,6 +474,10 @@ async function startPluginWorker(name: string) {
     }
   );
 
+  logger.verbose(
+    `[plugin-pool] spawned worker for "${name}" (pid: ${worker.pid}, socket: ${ipcPath})`
+  );
+
   // To make debugging easier and allow plugins to communicate things
   // like performance metrics, we pipe the stdout/stderr of the worker
   // to the main process.
@@ -529,18 +536,29 @@ async function startPluginWorker(name: string) {
       if (socket) {
         socket.unref();
         clearInterval(id);
+        logger.verbose(
+          `[plugin-pool] connected to worker for "${name}" (pid: ${worker.pid}) after ${attempts} attempt(s)`
+        );
         resolve({
           worker,
           socket,
         });
       } else if (attempts > 10000) {
         // daemon fails to start, the process probably exited
         // we print the logs and exit the client
-        reject('Failed to start plugin worker.');
+        clearInterval(id);
+        reject(new Error(`Failed to start plugin worker for plugin ${name}`));
       } else {
         attempts++;
       }
     }, 10);
+  }).finally(() => {
+    performance.mark(`start-plugin-worker-end:${name}`);
+    performance.measure(
+      `start-plugin-worker:${name}`,
+      `start-plugin-worker:${name}`,
+      `start-plugin-worker-end:${name}`
+    );
   });
 }