Fix multi-repo RPC deadlock via stderr redirect and dedicated GetObject pool (#7036)

Two independent deadlocks prevented `mod run` from completing across
multiple C# repositories:

1. **Stderr pipe buffer deadlock**: On macOS the kernel pipe buffer for
   stderr is 64KB. When the dotnet subprocess writes enough to stderr to
   fill this buffer, it blocks on the next write(), which deadlocks any
   in-progress RPC response on stdout. Fix: redirect stderr at the OS
   level via ProcessBuilder.redirectError() — to the log file when
   configured, or to /dev/null otherwise. This eliminates the pipe
   entirely so the subprocess never blocks on stderr.

2. **GetObject thread pool starvation**: GetObject.Handler submitted
   background tree traversal tasks to ForkJoinPool.commonPool(), which
   is the same pool used by the CLI for repo-level fork-join work. When
   repo tasks saturated the pool (blocked on semaphores or waiting for
   RPC responses), GetObject producers couldn't start, deadlocking the
   batch protocol. Fix: use a dedicated cached thread pool for tree
   traversal so GetObject producers are never starved by unrelated work.

Author: jkschneider

rewrite-core/src/main/java/org/openrewrite/rpc/RewriteRpcProcess.java

@@ -31,6 +31,7 @@
 import lombok.Setter;
 import org.jspecify.annotations.Nullable;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
@@ -46,6 +47,9 @@
  * A client for spawning and communicating with a subprocess that implements Rewrite RPC.
  */
 public class RewriteRpcProcess extends Thread {
+    private static final File DEV_NULL = new File(
+            System.getProperty("os.name").startsWith("Windows") ? "NUL" : "/dev/null");
+
     private final String[] command;
 
     @Setter
@@ -63,7 +67,8 @@ public class RewriteRpcProcess extends Thread {
 
     private final Map<String, String> environment = new LinkedHashMap<>();
 
-    private final StringBuffer accumulatedStderr = new StringBuffer();
+    @Setter
+    private @Nullable Path stderrRedirect;
 
     public RewriteRpcProcess(String... command) {
         this.command = command;
@@ -88,6 +93,11 @@ public void run() {
             if (workingDirectory != null) {
                 pb.directory(workingDirectory.toFile());
             }
+            if (stderrRedirect != null) {
+                pb.redirectError(ProcessBuilder.Redirect.appendTo(stderrRedirect.toFile()));
+            } else {
+                pb.redirectError(ProcessBuilder.Redirect.to(DEV_NULL));
+            }
             process = pb.start();
         } catch (IOException e) {
             throw new UncheckedIOException(e);
@@ -99,32 +109,9 @@ public void run() {
             return null;
         }
 
-        // Accumulate any available stderr
-        try {
-            InputStream errorStream = process.getErrorStream();
-            int available = errorStream.available();
-            if (available > 0) {
-                byte[] buffer = new byte[available];
-                int read = errorStream.read(buffer);
-                if (read > 0) {
-                    accumulatedStderr.append(new String(buffer, 0, read));
-                }
-            }
-        } catch (IOException | UnsupportedOperationException e) {
-            // Ignore errors reading stderr
-        }
-
         if (!process.isAlive()) {
             int exitCode = process.exitValue();
 
-            // Read any remaining stderr
-            try {
-                InputStream errorStream = process.getErrorStream();
-                accumulatedStderr.append(readFully(errorStream));
-            } catch (UnsupportedOperationException e) {
-                // Ignore errors reading final stderr
-            }
-
             // Read any remaining stdout
             String stdOutput = "";
             try {
@@ -135,12 +122,11 @@ public void run() {
             }
 
             String message = "RPC process shut down early with exit code " + exitCode;
-            String errorOutput = accumulatedStderr.toString();
             if (!stdOutput.isEmpty()) {
                 message += "\nStandard output:\n  " + stdOutput.replace("\n", "\n  ");
             }
-            if (!errorOutput.isEmpty()) {
-                message += "\nError output:\n  " + errorOutput.replace("\n", "\n  ");
+            if (stderrRedirect != null) {
+                message += "\nSee stderr log: " + stderrRedirect;
             }
             return new IllegalStateException(message.trim());
         }

rewrite-core/src/main/java/org/openrewrite/rpc/request/GetObject.java

@@ -44,7 +44,13 @@ public class GetObject implements RpcRequest {
 
     @RequiredArgsConstructor
     public static class Handler extends JsonRpcMethod<GetObject> {
-        private static final ExecutorService forkJoin = ForkJoinPool.commonPool();
+        // Dedicated pool for tree traversal so GetObject producers can't be starved
+        // by unrelated tasks (e.g. repo-level fork-join work) occupying the commonPool.
+        private static final ExecutorService TREE_TRAVERSAL_POOL = Executors.newCachedThreadPool(r -> {
+            Thread t = new Thread(r, "rpc-get-object-traversal");
+            t.setDaemon(true);
+            return t;
+        });
 
         private final AtomicInteger batchSize;
         private final Map<String, Object> remoteObjects;
@@ -77,7 +83,7 @@ protected List<RpcObjectData> handle(GetObject request) throws Exception {
                 Object before = remoteObjects.get(id);
 
                 RpcSendQueue sendQueue = new RpcSendQueue(batchSize.get(), batch::put, localRefs, request.getSourceFileType(), traceGetObject.get());
-                forkJoin.submit(() -> {
+                TREE_TRAVERSAL_POOL.submit(() -> {
                     try {
                         sendQueue.send(after, before, null);
 

rewrite-csharp/src/main/java/org/openrewrite/csharp/rpc/CSharpRewriteRpc.java

@@ -323,6 +323,7 @@ private CSharpRewriteRpc startProcess(Stream<@Nullable String> cmd) {
             if (workingDirectory != null) {
                 process.setWorkingDirectory(workingDirectory);
             }
+            process.setStderrRedirect(log);
 
             process.environment().putAll(environment);
 

rewrite-javascript/src/main/java/org/openrewrite/javascript/rpc/JavaScriptRewriteRpc.java

@@ -382,6 +382,7 @@ public JavaScriptRewriteRpc get() {
             if (workingDirectory != null) {
                 process.setWorkingDirectory(workingDirectory);
             }
+            process.setStderrRedirect(log);
 
             process.environment().putAll(environment);
             // caller-provided options, if any, are taking precedence over the options baked above

rewrite-python/src/main/java/org/openrewrite/python/rpc/PythonRewriteRpc.java

@@ -581,6 +581,7 @@ public PythonRewriteRpc get() {
             if (workingDirectory != null) {
                 process.setWorkingDirectory(workingDirectory);
             }
+            process.setStderrRedirect(log);
 
             process.environment().putAll(environment);