Treat all packages listed in requirements.txt as direct dependencies (#7320)

* Treat all packages listed in requirements.txt as direct dependencies

Previously, `dependenciesFromResolved()` only treated graph-root packages
(those not depended on by any other installed package) as direct. This
caused packages like `aiohttp`, `cryptography`, `Jinja2` etc. to be
classified as transitive when they were explicitly pinned in the
requirements.txt but also happened to be dependencies of other listed
packages. As a result, vulnerability fix recipes skipped upgrading them.

Now, the parser scans the requirements.txt file content and passes the
declared package names to `dependenciesFromResolved()`, ensuring every
explicitly listed package is treated as a direct dependency regardless
of the dependency graph.

Fixes moderneinc/customer-requests#2157

* Add test for mixed declared/undeclared transitive dependencies

Verifies that packages listed in requirements.txt are treated as direct
while packages only present in the freeze output (true transitives) are
excluded from the dependencies list.

* Remove excessive comments

* Reuse PyProjectHelper.extractPackageName for parsing declared packages

Author: Jenson3210

rewrite-python/src/main/java/org/openrewrite/python/RequirementsTxtParser.java

@@ -19,6 +19,7 @@
 import org.openrewrite.ExecutionContext;
 import org.openrewrite.Parser;
 import org.openrewrite.SourceFile;
+import org.openrewrite.python.internal.PyProjectHelper;
 import org.openrewrite.python.marker.PythonResolutionResult;
 import org.openrewrite.python.marker.PythonResolutionResult.Dependency;
 import org.openrewrite.python.marker.PythonResolutionResult.PackageManager;
@@ -85,7 +86,8 @@ public Stream<SourceFile> parseInputs(Iterable<Input> sources, @Nullable Path re
                 return sf;
             }
 
-            List<Dependency> deps = dependenciesFromResolved(resolvedDeps);
+            List<Dependency> deps = dependenciesFromResolved(resolvedDeps,
+                    parseDeclaredPackageNames(text.getText()));
 
             PythonResolutionResult marker = new PythonResolutionResult(
                     randomId(),
@@ -142,7 +144,11 @@ static List<ResolvedDependency> parseFreezeLines(String freezeContent) {
      * are treated as direct so that client code traversing {@code getDependencies()} finds every package.
      */
     public static List<Dependency> dependenciesFromResolved(List<ResolvedDependency> resolved) {
-        // Collect all packages that appear as a transitive dependency of another package
+        return dependenciesFromResolved(resolved, Collections.emptySet());
+    }
+
+    public static List<Dependency> dependenciesFromResolved(List<ResolvedDependency> resolved,
+                                                            Set<String> declaredPackageNames) {
         Set<String> transitive = new HashSet<>();
         for (ResolvedDependency r : resolved) {
             if (r.getDependencies() != null) {
@@ -154,13 +160,31 @@ public static List<Dependency> dependenciesFromResolved(List<ResolvedDependency>
 
         List<Dependency> deps = new ArrayList<>();
         for (ResolvedDependency r : resolved) {
-            if (transitive.isEmpty() || !transitive.contains(PythonResolutionResult.normalizeName(r.getName()))) {
+            String normalizedName = PythonResolutionResult.normalizeName(r.getName());
+            if (transitive.isEmpty() ||
+                    !transitive.contains(normalizedName) ||
+                    declaredPackageNames.contains(normalizedName)) {
                 deps.add(new Dependency(r.getName(), "==" + r.getVersion(), null, null, r));
             }
         }
         return deps;
     }
 
+    static Set<String> parseDeclaredPackageNames(String requirementsTxtContent) {
+        Set<String> names = new HashSet<>();
+        for (String line : requirementsTxtContent.split("\n")) {
+            String trimmed = line.trim();
+            if (trimmed.isEmpty() || trimmed.startsWith("#") || trimmed.startsWith("-")) {
+                continue;
+            }
+            String name = PyProjectHelper.extractPackageName(trimmed);
+            if (name != null) {
+                names.add(PythonResolutionResult.normalizeName(name));
+            }
+        }
+        return names;
+    }
+
     /**
      * Link transitive dependencies by reading installed package METADATA files from site-packages.
      * Uses a two-pass approach: first builds a name→entry map, then reads each package's

rewrite-python/src/main/java/org/openrewrite/python/internal/PyProjectHelper.java

@@ -65,7 +65,7 @@ public static String normalizeVersionConstraint(String version) {
         int end = 0;
         while (end < trimmed.length()) {
             char c = trimmed.charAt(end);
-            if (c == '[' || c == '>' || c == '<' || c == '=' || c == '!' || c == '~' || c == ';' || c == ' ') {
+            if (c == '[' || c == '>' || c == '<' || c == '=' || c == '!' || c == '~' || c == ';' || c == ' ' || c == '@') {
                 break;
             }
             end++;

rewrite-python/src/test/java/org/openrewrite/python/RequirementsTxtParserTest.java

@@ -33,6 +33,7 @@
 import java.nio.file.Paths;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -107,6 +108,57 @@ void dependenciesFromResolvedExcludesTransitives() {
         assertThat(deps.get(0).getResolved()).isSameAs(requests);
     }
 
+    @Test
+    void dependenciesFromResolvedTreatsDeclaredPackagesAsDirect() {
+        ResolvedDependency certifi = new ResolvedDependency("certifi", "2024.2.2", null, null);
+        ResolvedDependency requests = new ResolvedDependency("requests", "2.31.0", null, List.of(certifi));
+
+        List<ResolvedDependency> resolved = List.of(certifi, requests);
+        Set<String> declared = RequirementsTxtParser.parseDeclaredPackageNames(
+                "certifi==2024.2.2\nrequests==2.31.0\n");
+        List<Dependency> deps = RequirementsTxtParser.dependenciesFromResolved(resolved, declared);
+
+        assertThat(deps).hasSize(2);
+        assertThat(deps.get(0).getName()).isEqualTo("certifi");
+        assertThat(deps.get(1).getName()).isEqualTo("requests");
+    }
+
+    @Test
+    void declaredPackagesAreDirectAndUndeclaredTransitivesAreExcluded() {
+        ResolvedDependency urllib3 = new ResolvedDependency("urllib3", "2.2.1", null, null);
+        ResolvedDependency charsetNormalizer = new ResolvedDependency("charset-normalizer", "3.3.2", null, null);
+        ResolvedDependency certifi = new ResolvedDependency("certifi", "2024.2.2", null, null);
+        ResolvedDependency requests = new ResolvedDependency("requests", "2.31.0", null,
+                List.of(certifi, urllib3, charsetNormalizer));
+
+        List<ResolvedDependency> resolved = List.of(urllib3, charsetNormalizer, certifi, requests);
+        Set<String> declared = RequirementsTxtParser.parseDeclaredPackageNames(
+                "requests==2.31.0\ncertifi==2024.2.2\n");
+        List<Dependency> deps = RequirementsTxtParser.dependenciesFromResolved(resolved, declared);
+
+        assertThat(deps).hasSize(2);
+        assertThat(deps.stream().map(Dependency::getName))
+                .containsExactly("certifi", "requests");
+    }
+
+    @Test
+    void parseDeclaredPackageNamesExtractsNames() {
+        Set<String> names = RequirementsTxtParser.parseDeclaredPackageNames("""
+                # This is a comment
+                requests>=2.28.0
+                certifi==2024.2.2
+                charset-normalizer<4,>=2
+                Jinja2~=3.1.5
+                -r other-requirements.txt
+                aiohttp==3.13.3
+
+                langchain-core==1.2.12
+                """);
+        assertThat(names).containsExactlyInAnyOrder(
+                "requests", "certifi", "charset_normalizer", "jinja2",
+                "aiohttp", "langchain_core");
+    }
+
     @Test
     void linkDependenciesFromMetadataBuildsGraph(@TempDir Path tempDir) throws IOException {
         // Create a fake site-packages with METADATA files