PipRecipeBundleResolver: pip install on the server side

Mirrors the JS RPC server's design where `npm install` runs server-side
during InstallRecipes. Today the Python RPC server's
handle_install_recipes assumes the package is already importable; if it
isn't, prepareRecipe later throws "Recipe not found". The JVM-side
PipRecipeBundleResolver doesn't pip install either, so the contract was
implicit: "some other process must have pre-installed this package",
which is fragile and forced downstream callers to duplicate install
logic.

Fix on the Python side:
- New `--recipe-install-dir` argv stores the install target globally.
- `_pip_install_recipe_package` shells out to `python -m pip install
  --target <dir> <package>==<version>` and primes sys.path.
- `handle_install_recipes`'s package-spec branch invokes it when the
  requested package isn't already importable at the requested version,
  before the existing import + activate flow.

Fix on the Java side:
- `PythonRewriteRpc` passes `--recipe-install-dir=<path>` to the
  Python process when the builder's `recipeInstallDir` is set. The
  existing PYTHONPATH wiring is unchanged, so a package that's already
  installed remains importable as before.

The local-path branch (caller passes a directory containing an already-
unpacked recipe package) is unchanged.

Author: jkschneider

rewrite-python/rewrite/src/rewrite/rpc/server.py

@@ -75,6 +75,12 @@
 # Set REWRITE_PYTHON_VERSION to "2" or "2.7" to parse Python 2 code
 _python_version = os.environ.get("REWRITE_PYTHON_VERSION", "3")
 
+# Directory where recipe pip packages are installed (set via --recipe-install-dir
+# at startup; mirrors how the JS RPC server treats `installDir` on the npm side).
+# When set, an InstallRecipes RPC for a not-yet-importable package will pip
+# install it into this dir before activating.
+_recipe_install_dir: Optional[Path] = None
+
 
 def _next_request_id() -> int:
     """Generate a unique request ID for outgoing requests."""
@@ -627,12 +633,54 @@ def _get_marketplace():
     return _marketplace
 
 
+def _is_package_installed(package_name: str, version: Optional[str]) -> bool:
+    """True if `package_name` is importable at `version` (or any version when
+    `version` is None). Uses importlib.metadata so we don't have to import the
+    package itself just to check."""
+    try:
+        import importlib.metadata
+        installed = importlib.metadata.version(package_name)
+    except Exception:
+        return False
+    return version is None or installed == version
+
+
+def _pip_install_recipe_package(package_name: str, version: Optional[str], target_dir: Path) -> None:
+    """pip install `package_name[==version]` into `target_dir` and make it
+    visible to subsequent imports. Raises RuntimeError on pip failure.
+
+    Mirrors the JS RPC server's server-side `npm install` for unknown recipe
+    packages; the JVM caller doesn't need to pre-install."""
+    import importlib
+    import subprocess
+
+    target_dir.mkdir(parents=True, exist_ok=True)
+    spec = f"{package_name}=={version}" if version else package_name
+    cmd = [sys.executable, "-m", "pip", "install", "--target", str(target_dir), spec]
+    logger.info(f"pip install: {' '.join(cmd)}")
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"pip install failed for {spec} (target={target_dir}):\n{result.stderr}"
+        )
+
+    # Make the freshly-installed package importable. The Java side already
+    # added target_dir to PYTHONPATH at process start, but if it's missing from
+    # sys.path (e.g. recipeInstallDir was set after launch) add it here too.
+    target_str = str(target_dir.resolve())
+    if target_str not in sys.path:
+        sys.path.insert(0, target_str)
+    importlib.invalidate_caches()
+
+
 def handle_install_recipes(params: dict) -> dict:
     """Handle an InstallRecipes RPC request.
 
-    Activates a recipe package in the marketplace. The package should already be
-    installed by the caller (e.g., via pip install --target). This handler discovers
-    and activates the package's recipes.
+    Activates a recipe package in the marketplace. When `--recipe-install-dir`
+    is configured at server startup, a package spec that isn't already installed
+    is pip-installed into that directory before activation; otherwise the
+    package must have been installed by the caller (e.g. via `pip install
+    --target`).
 
     Args:
         params: Dict containing either:
@@ -675,16 +723,21 @@ def handle_install_recipes(params: dict) -> dict:
             recipes_added = len(added)
 
     elif isinstance(recipes, dict):
-        # Package spec with name and optional version - package should already be installed
+        # Package spec with name and optional version. Pip install into the
+        # configured `--recipe-install-dir` if the package isn't already
+        # importable at the requested version. Mirrors how the JS RPC server
+        # shells out to `npm install` for unknown npm recipe packages.
         package_name = recipes.get('packageName')
         version = recipes.get('version')
 
         if not package_name:
             raise ValueError("Package name is required")
 
+        if _recipe_install_dir is not None and not _is_package_installed(package_name, version):
+            _pip_install_recipe_package(package_name, version, _recipe_install_dir)
+
         logger.info(f"Activating recipes package: {package_name}")
 
-        # Get the installed version
         try:
             import importlib.metadata
             installed_version = importlib.metadata.version(package_name)
@@ -1640,8 +1693,13 @@ def main():
     parser.add_argument('--log-file', help='Log file path')
     parser.add_argument('--metrics-csv', help='Metrics CSV output path')
     parser.add_argument('--trace-rpc-messages', action='store_true', help='Enable RPC message tracing')
+    parser.add_argument('--recipe-install-dir', help='Directory where recipe pip packages are installed')
     args = parser.parse_args()
 
+    if args.recipe_install_dir:
+        global _recipe_install_dir
+        _recipe_install_dir = Path(args.recipe_install_dir)
+
     if args.log_file:
         file_handler = logging.FileHandler(args.log_file)
         file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))

rewrite-python/rewrite/tests/rpc/test_server.py

@@ -26,6 +26,49 @@ def fake_parse_python_source(source, path="<unknown>", relative_to=None, ty_clie
     assert (tmp_path / "pkg" / "__init__.py").read_text(encoding="utf-8") == ""
 
 
+def test_handle_install_recipes_pip_installs_when_target_configured(tmp_path, monkeypatch):
+    """When `--recipe-install-dir` is configured and the package isn't already
+    importable, handle_install_recipes should shell out to `pip install --target
+    <dir> packageName==version` before activating, mirroring the JS RPC server's
+    server-side `npm install`. The JVM caller must not have to pre-install."""
+    import rewrite.rpc.server as server
+    import subprocess
+
+    install_dir = tmp_path / "recipes"
+    monkeypatch.setattr(server, "_recipe_install_dir", install_dir)
+    monkeypatch.setattr(server, "_marketplace", None)
+
+    # Stand in for pip — capture the command, fake a successful run.
+    captured = {}
+
+    class FakeCompletedProcess:
+        returncode = 0
+        stdout = ""
+        stderr = ""
+
+    def fake_run(cmd, capture_output=False, text=False):
+        captured["cmd"] = cmd
+        return FakeCompletedProcess()
+
+    monkeypatch.setattr(subprocess, "run", fake_run)
+
+    # The package "definitely-not-installed-pkg" doesn't exist; _is_package_installed
+    # returns False, which triggers the pip install path. We don't go further —
+    # _import_and_activate_package would raise after our fake pip — so call the
+    # helper directly to assert the install command shape.
+    server._pip_install_recipe_package(
+        "openrewrite-recipes-python", "1.2.3", install_dir
+    )
+
+    assert install_dir.exists()
+    assert captured["cmd"][1:] == [
+        "-m", "pip", "install",
+        "--target", str(install_dir),
+        "openrewrite-recipes-python==1.2.3",
+    ]
+    assert str(install_dir.resolve()) in __import__("sys").path
+
+
 def test_recipe_descriptor_to_dict_emits_all_collection_keys():
     from rewrite.recipe import RecipeDescriptor
     from rewrite.rpc.server import _recipe_descriptor_to_dict

rewrite-python/src/main/java/org/openrewrite/python/rpc/PythonRewriteRpc.java

@@ -599,7 +599,8 @@ public PythonRewriteRpc get() {
                     "-m", "rewrite.rpc.server",
                     log == null ? null : "--log-file=" + log.toAbsolutePath().normalize(),
                     metricsCsv == null ? null : "--metrics-csv=" + metricsCsv.toAbsolutePath().normalize(),
-                    traceRpcMessages ? "--trace-rpc-messages" : null
+                    traceRpcMessages ? "--trace-rpc-messages" : null,
+                    recipeInstallDir == null ? null : "--recipe-install-dir=" + recipeInstallDir.toAbsolutePath().normalize()
             );
 
             String[] cmdArr = cmd.filter(Objects::nonNull).toArray(String[]::new);