Python: Support legacy project manifests (setup.cfg etc.) (#6716)

* Add requirements.txt parser with dependency resolution and insight support

Parse requirements.txt files into PlainText with a PythonResolutionResult
marker containing declared dependencies and optionally resolved versions
via uv pip freeze. Extend DependencyInsight recipe to search requirements.txt
in addition to pyproject.toml. Include manifest files (pyproject.toml or
requirements.txt) in ParseProject results from the Java side.

* Add setup.cfg and setup.py parser support with dependency resolution

Add SetupCfgParser (PlainText-based) for setup.cfg files and setup.py
marker attachment in PythonRewriteRpc. Both use uv pip install + freeze
for dependency resolution, with all resolved packages treated as
dependencies since pip freeze doesn't distinguish direct from transitive.

- SetupCfgParser: accepts setup.cfg, delegates to PlainTextParser,
  resolves via DependencyWorkspace.getOrCreateSetuptoolsWorkspace()
- PythonRewriteRpc: setup.cfg added to parseManifest() priority chain
  (pyproject.toml > setup.cfg > requirements.txt); setup.py-only
  projects get marker attached to Py.CompilationUnit in RPC stream
- DependencyInsight: extended for setup.cfg (PlainText) and setup.py
  (Py.CompilationUnit with marker)
- Assertions: setupCfg() test helpers added
- RequirementsTxtParser: simplified to derive dependencies from freeze
  output using shared dependenciesFromResolved() method
- DependencyWorkspace: added getOrCreateSetuptoolsWorkspace() using
  uv pip install <projectDir>

* Link transitive dependency graph from installed package METADATA

Read Requires-Dist entries from .dist-info/METADATA files in
site-packages to build the dependency graph. This allows
distinguishing direct from transitive dependencies: only root
packages (those not depended on by any other installed package)
are included in the declared dependencies list.

* Check for PythonResolutionResult marker in isAcceptable for all file types

* Add missing constraintDependencies and overrideDependencies params to PythonResolutionResult constructors

* Fix review items: Windows site-packages, dist-info version matching, extra marker regex, SetupCfgParser test coverage

Author: knutwannheden

rewrite-python/src/integTest/java/org/openrewrite/python/ParseProjectIntegTest.java

@@ -24,6 +24,7 @@
 import org.openrewrite.SourceFile;
 import org.openrewrite.python.marker.PythonResolutionResult;
 import org.openrewrite.python.rpc.PythonRewriteRpc;
+import org.openrewrite.text.PlainText;
 import org.openrewrite.toml.tree.Toml;
 
 import java.io.IOException;
@@ -196,6 +197,63 @@ void includesPyprojectToml() throws IOException {
         assertThat(pyproject.getMarkers().findFirst(PythonResolutionResult.class)).isPresent();
     }
 
+    @Test
+    @Timeout(value = 60, unit = TimeUnit.SECONDS)
+    void includesRequirementsTxt() throws IOException {
+        Path projectDir = tempDir.resolve("with_requirements");
+        Files.createDirectories(projectDir);
+
+        Files.writeString(projectDir.resolve("main.py"), "x = 1");
+        Files.writeString(projectDir.resolve("requirements.txt"), """
+                requests>=2.28.0
+                click>=8.0
+                """);
+
+        List<SourceFile> sources = client()
+                .parseProject(projectDir, new InMemoryExecutionContext())
+                .collect(Collectors.toList());
+
+        assertThat(sources)
+                .extracting(sf -> sf.getSourcePath().getFileName().toString())
+                .contains("main.py", "requirements.txt");
+
+        SourceFile reqsTxt = sources.stream()
+                .filter(sf -> sf.getSourcePath().getFileName().toString().equals("requirements.txt"))
+                .findFirst()
+                .orElseThrow();
+        assertThat(reqsTxt).isInstanceOf(PlainText.class);
+        assertThat(reqsTxt.getMarkers().findFirst(PythonResolutionResult.class)).isPresent();
+    }
+
+    @Test
+    @Timeout(value = 60, unit = TimeUnit.SECONDS)
+    void pyprojectTomlTakesPriorityOverRequirementsTxt() throws IOException {
+        Path projectDir = tempDir.resolve("both_manifests");
+        Files.createDirectories(projectDir);
+
+        Files.writeString(projectDir.resolve("main.py"), "x = 1");
+        Files.writeString(projectDir.resolve("pyproject.toml"), """
+                [project]
+                name = "myapp"
+                version = "1.0.0"
+                dependencies = ["requests>=2.28.0"]
+                """);
+        Files.writeString(projectDir.resolve("requirements.txt"), """
+                requests>=2.28.0
+                """);
+
+        List<SourceFile> sources = client()
+                .parseProject(projectDir, new InMemoryExecutionContext())
+                .collect(Collectors.toList());
+
+        // pyproject.toml should be included but not requirements.txt
+        assertThat(sources)
+                .extracting(sf -> sf.getSourcePath().getFileName().toString())
+                .contains("pyproject.toml")
+                .doesNotContain("requirements.txt");
+    }
+
+
     private PythonRewriteRpc client() {
         return PythonRewriteRpc.getOrStart();
     }

rewrite-python/src/main/java/org/openrewrite/python/Assertions.java

@@ -20,6 +20,7 @@
 import org.openrewrite.python.tree.Py;
 import org.openrewrite.test.SourceSpec;
 import org.openrewrite.test.SourceSpecs;
+import org.openrewrite.text.PlainText;
 import org.openrewrite.toml.tree.Toml;
 
 import java.io.IOException;
@@ -145,6 +146,84 @@ public static SourceSpecs uvLock(@Language("toml") @Nullable String before,
         return toml;
     }
 
+    public static SourceSpecs requirementsTxt(@Nullable String before) {
+        return requirementsTxt(before, s -> {
+        });
+    }
+
+    public static SourceSpecs requirementsTxt(@Nullable String before,
+                                               Consumer<SourceSpec<PlainText>> spec) {
+        SourceSpec<PlainText> text = new SourceSpec<>(
+                PlainText.class, null, RequirementsTxtParser.builder(), before,
+                SourceSpec.ValidateSource.noop,
+                ctx -> {
+                }
+        );
+        text.path("requirements.txt");
+        spec.accept(text);
+        return text;
+    }
+
+    public static SourceSpecs requirementsTxt(@Nullable String before,
+                                               @Nullable String after) {
+        return requirementsTxt(before, after, s -> {
+        });
+    }
+
+    public static SourceSpecs requirementsTxt(@Nullable String before,
+                                               @Nullable String after,
+                                               Consumer<SourceSpec<PlainText>> spec) {
+        SourceSpec<PlainText> text = new SourceSpec<>(
+                PlainText.class, null, RequirementsTxtParser.builder(), before,
+                SourceSpec.ValidateSource.noop,
+                ctx -> {
+                }
+        );
+        text.path("requirements.txt");
+        text.after(s -> after);
+        spec.accept(text);
+        return text;
+    }
+
+    public static SourceSpecs setupCfg(@Nullable String before) {
+        return setupCfg(before, s -> {
+        });
+    }
+
+    public static SourceSpecs setupCfg(@Nullable String before,
+                                        Consumer<SourceSpec<PlainText>> spec) {
+        SourceSpec<PlainText> text = new SourceSpec<>(
+                PlainText.class, null, SetupCfgParser.builder(), before,
+                SourceSpec.ValidateSource.noop,
+                ctx -> {
+                }
+        );
+        text.path("setup.cfg");
+        spec.accept(text);
+        return text;
+    }
+
+    public static SourceSpecs setupCfg(@Nullable String before,
+                                        @Nullable String after) {
+        return setupCfg(before, after, s -> {
+        });
+    }
+
+    public static SourceSpecs setupCfg(@Nullable String before,
+                                        @Nullable String after,
+                                        Consumer<SourceSpec<PlainText>> spec) {
+        SourceSpec<PlainText> text = new SourceSpec<>(
+                PlainText.class, null, SetupCfgParser.builder(), before,
+                SourceSpec.ValidateSource.noop,
+                ctx -> {
+                }
+        );
+        text.path("setup.cfg");
+        text.after(s -> after);
+        spec.accept(text);
+        return text;
+    }
+
     public static SourceSpecs python(@Language("py") @Nullable String before) {
         return python(before, s -> {
         });

rewrite-python/src/main/java/org/openrewrite/python/DependencyWorkspace.java

@@ -16,6 +16,7 @@
 package org.openrewrite.python;
 
 import lombok.experimental.UtilityClass;
+import org.jspecify.annotations.Nullable;
 import org.openrewrite.python.internal.UvExecutor;
 
 import java.io.IOException;
@@ -39,7 +40,7 @@
  * to enable proper type attribution via ty LSP.
  */
 @UtilityClass
-class DependencyWorkspace {
+public class DependencyWorkspace {
     private static final Path WORKSPACE_BASE = Paths.get(
             System.getProperty("java.io.tmpdir"),
             "openrewrite-python-workspaces"
@@ -132,6 +133,197 @@ static Path getOrCreateWorkspace(String pyprojectContent) {
         }
     }
 
+    /**
+     * Gets or creates a workspace directory for a requirements.txt file.
+     * Returns null (graceful degradation) when uv is unavailable.
+     *
+     * @param requirementsContent The complete requirements.txt content
+     * @param originalFilePath    The original file path on disk (supports -r includes), or null
+     * @return Path to the workspace directory, or null if uv is unavailable
+     */
+    static @Nullable Path getOrCreateRequirementsWorkspace(String requirementsContent,
+                                                            @Nullable Path originalFilePath) {
+        String uvPath = UvExecutor.findUvExecutable();
+        if (uvPath == null) {
+            return null;
+        }
+
+        String hash = "req-" + hashContent(requirementsContent);
+
+        // Check in-memory cache
+        Path cached = cache.get(hash);
+        if (cached != null && isRequirementsWorkspaceValid(cached)) {
+            return cached;
+        }
+
+        // Check disk cache
+        Path workspaceDir = WORKSPACE_BASE.resolve(hash);
+        if (isRequirementsWorkspaceValid(workspaceDir)) {
+            cache.put(hash, workspaceDir);
+            return workspaceDir;
+        }
+
+        // Create new workspace
+        try {
+            Files.createDirectories(WORKSPACE_BASE);
+
+            Path tempDir = Files.createTempDirectory(WORKSPACE_BASE, hash + ".tmp-");
+
+            try {
+                // Create virtualenv
+                runCommandWithPath(tempDir, uvPath, "venv");
+
+                // Install dependencies from requirements file
+                Path reqFile;
+                if (originalFilePath != null && Files.exists(originalFilePath)) {
+                    reqFile = originalFilePath;
+                } else {
+                    reqFile = tempDir.resolve("requirements.txt");
+                    Files.write(reqFile, requirementsContent.getBytes(StandardCharsets.UTF_8));
+                }
+                runCommandWithPath(tempDir, uvPath, "pip", "install", "-r", reqFile.toString());
+
+                // Capture freeze output BEFORE installing ty
+                UvExecutor.RunResult freezeResult = UvExecutor.run(tempDir, uvPath, "pip", "freeze");
+                if (freezeResult.isSuccess()) {
+                    Files.write(
+                            tempDir.resolve("freeze.txt"),
+                            freezeResult.getStdout().getBytes(StandardCharsets.UTF_8)
+                    );
+                }
+
+                // Install ty for type stubs (after freeze so it's not in the dep model)
+                runCommandWithPath(tempDir, uvPath, "pip", "install", "ty");
+
+                // Move to final location
+                try {
+                    Files.move(tempDir, workspaceDir);
+                } catch (IOException e) {
+                    if (isRequirementsWorkspaceValid(workspaceDir)) {
+                        cleanupDirectory(tempDir);
+                    } else {
+                        throw e;
+                    }
+                }
+
+                cache.put(hash, workspaceDir);
+                return workspaceDir;
+
+            } catch (Exception e) {
+                cleanupDirectory(tempDir);
+                return null;
+            }
+
+        } catch (IOException e) {
+            return null;
+        }
+    }
+
+    /**
+     * Gets or creates a workspace directory for a setuptools project (setup.cfg / setup.py).
+     * Uses {@code uv pip install <projectDir>} to install the project and its dependencies.
+     * Returns null (graceful degradation) when uv is unavailable.
+     *
+     * @param manifestContent The setup.cfg (or setup.py) content for hashing
+     * @param projectDir      The project directory to install from, or null
+     * @return Path to the workspace directory, or null if uv is unavailable
+     */
+    public static @Nullable Path getOrCreateSetuptoolsWorkspace(String manifestContent,
+                                                                @Nullable Path projectDir) {
+        String uvPath = UvExecutor.findUvExecutable();
+        if (uvPath == null) {
+            return null;
+        }
+
+        String hash = "setup-" + hashContent(manifestContent);
+
+        // Check in-memory cache
+        Path cached = cache.get(hash);
+        if (cached != null && isRequirementsWorkspaceValid(cached)) {
+            return cached;
+        }
+
+        // Check disk cache
+        Path workspaceDir = WORKSPACE_BASE.resolve(hash);
+        if (isRequirementsWorkspaceValid(workspaceDir)) {
+            cache.put(hash, workspaceDir);
+            return workspaceDir;
+        }
+
+        if (projectDir == null || !Files.isDirectory(projectDir)) {
+            return null;
+        }
+
+        // Create new workspace
+        try {
+            Files.createDirectories(WORKSPACE_BASE);
+
+            Path tempDir = Files.createTempDirectory(WORKSPACE_BASE, hash + ".tmp-");
+
+            try {
+                // Create virtualenv
+                runCommandWithPath(tempDir, uvPath, "venv");
+
+                // Install from the project directory
+                runCommandWithPath(tempDir, uvPath, "pip", "install", projectDir.toString());
+
+                // Capture freeze output BEFORE installing ty
+                UvExecutor.RunResult freezeResult = UvExecutor.run(tempDir, uvPath, "pip", "freeze");
+                if (freezeResult.isSuccess()) {
+                    Files.write(
+                            tempDir.resolve("freeze.txt"),
+                            freezeResult.getStdout().getBytes(StandardCharsets.UTF_8)
+                    );
+                }
+
+                // Install ty for type stubs (after freeze so it's not in the dep model)
+                runCommandWithPath(tempDir, uvPath, "pip", "install", "ty");
+
+                // Move to final location
+                try {
+                    Files.move(tempDir, workspaceDir);
+                } catch (IOException e) {
+                    if (isRequirementsWorkspaceValid(workspaceDir)) {
+                        cleanupDirectory(tempDir);
+                    } else {
+                        throw e;
+                    }
+                }
+
+                cache.put(hash, workspaceDir);
+                return workspaceDir;
+
+            } catch (Exception e) {
+                cleanupDirectory(tempDir);
+                return null;
+            }
+
+        } catch (IOException e) {
+            return null;
+        }
+    }
+
+    static String readFreezeOutput(Path workspaceDir) {
+        try {
+            return new String(Files.readAllBytes(workspaceDir.resolve("freeze.txt")), StandardCharsets.UTF_8);
+        } catch (IOException e) {
+            return "";
+        }
+    }
+
+    private static boolean isRequirementsWorkspaceValid(Path workspaceDir) {
+        return Files.exists(workspaceDir) &&
+                Files.isDirectory(workspaceDir.resolve(".venv")) &&
+                Files.exists(workspaceDir.resolve("freeze.txt"));
+    }
+
+    private static void runCommandWithPath(Path dir, String uvPath, String... args) throws IOException, InterruptedException {
+        UvExecutor.RunResult result = UvExecutor.run(dir, uvPath, args);
+        if (!result.isSuccess()) {
+            throw new RuntimeException("uv " + String.join(" ", args) + " failed with exit code: " + result.getExitCode());
+        }
+    }
+
     private static void runCommand(Path dir, String... command) throws IOException, InterruptedException {
         String uvPath = UvExecutor.findUvExecutable();
         if (uvPath == null) {
@@ -200,7 +392,13 @@ private static void initializeCacheFromDisk() {
             Files.list(WORKSPACE_BASE)
                     .filter(Files::isDirectory)
                     .filter(dir -> !dir.getFileName().toString().contains(".tmp-"))
-                    .filter(DependencyWorkspace::isWorkspaceValid)
+                    .filter(dir -> {
+                        String name = dir.getFileName().toString();
+                        if (name.startsWith("req-") || name.startsWith("setup-")) {
+                            return isRequirementsWorkspaceValid(dir);
+                        }
+                        return isWorkspaceValid(dir);
+                    })
                     .sorted((a, b) -> {
                         try {
                             return Files.getLastModifiedTime(a).compareTo(Files.getLastModifiedTime(b));