Skip to content

Commit 9da74df

Browse files
knutwannhedenknutwannheden
authored
Python: Improve dependency recipe lock file handling and version normalization (#6752)
* Normalize bare version constraints in Python dependency recipes Bare version strings like `2.28.0` passed to `AddDependency`, `UpgradeDependencyVersion`, `UpgradeTransitiveDependencyVersion`, and `ChangeDependency` were concatenated directly with the package name, producing invalid PEP 508 specs such as `requests2.28.0`. A shared `PyProjectHelper.normalizeVersionConstraint()` now prefixes `>=` when no comparison operator is present. Also parse `uv.lock` in `PythonRewriteRpc.parseProject()` via the standard `TomlParser` so that lock files are included in the source set and can be updated by dependency-management recipes. * Fix handle_parse_project not passing relativeTo to parse_python_file The Python RPC server's handle_parse_project extracted relative_to from the request but never forwarded it, producing absolute source paths. Now defaults to project_path (matching the JS implementation) and passes it through so source paths are relative. Also strengthened ParseProjectIntegTest to assert full relative paths. * Seed existing uv.lock during lock regeneration for minimal updates When regenerating uv.lock after dependency changes, seed the temp directory with the existing lock file so `uv lock` performs a minimal update rather than re-resolving every dependency from scratch. Also extend TomlParser to accept uv.lock files, allowing the scanner phase of dependency recipes to capture existing lock contents. * Share lock file state across recipes via ExecutionContextView Move lock file maps (updatedLockFiles, existingLockContents) from per-recipe accumulators into a shared PythonDependencyExecutionContextView. This allows sequential recipes in a CompositeRecipe to correctly build on each other's lock regeneration results. Also extract maybeUpdateUvLock helper in PyProjectHelper that handles idempotent uv.lock updates with content normalization for round-trip stability across recipe cycles.
1 parent 6df8209 commit 9da74df

16 files changed

Lines changed: 474 additions & 70 deletions

File tree

‎rewrite-python/rewrite/src/rewrite/rpc/server.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -350,7 +350,7 @@ def handle_parse_project(params: dict) -> List[dict]:
350350

351351
project_path = params.get('projectPath', '.')
352352
exclusions = params.get('exclusions', ['__pycache__', '.venv', 'venv', '.git', '.tox', '*.egg-info'])
353-
relative_to = params.get('relativeTo')
353+
relative_to = params.get('relativeTo') or project_path
354354

355355
results = []
356356

@@ -362,7 +362,7 @@ def handle_parse_project(params: dict) -> List[dict]:
362362
if file.endswith('.py'):
363363
path = os.path.join(root, file)
364364
try:
365-
result = parse_python_file(path)
365+
result = parse_python_file(path, relative_to)
366366
results.append(result)
367367
except Exception as e:
368368
logger.error(f"Error parsing {path}: {e}")

‎rewrite-python/src/integTest/java/org/openrewrite/python/ParseProjectIntegTest.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,10 @@ void parsesNestedDirectories() throws IOException {
112112
.collect(Collectors.toList());
113113

114114
assertThat(sources).hasSize(2);
115+
// Source paths must be relative to the project directory
116+
assertThat(sources)
117+
.extracting(sf -> sf.getSourcePath().toString())
118+
.containsExactlyInAnyOrder("top.py", "subpackage/nested.py");
115119
}
116120

117121
@Test

‎rewrite-python/src/main/java/org/openrewrite/python/AddDependency.java‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.openrewrite.*;
2222
import org.openrewrite.marker.Markers;
2323
import org.openrewrite.python.internal.PyProjectHelper;
24+
import org.openrewrite.python.internal.PythonDependencyExecutionContextView;
2425
import org.openrewrite.python.marker.PythonResolutionResult;
2526
import org.openrewrite.toml.TomlIsoVisitor;
2627
import org.openrewrite.toml.tree.Space;
@@ -95,7 +96,6 @@ public String getDescription() {
9596

9697
static class Accumulator {
9798
final Set<String> projectsToUpdate = new HashSet<>();
98-
final Map<String, String> updatedLockFiles = new HashMap<>();
9999
}
100100

101101
@Override
@@ -108,7 +108,16 @@ public TreeVisitor<?, ExecutionContext> getScanner(Accumulator acc) {
108108
return new TomlIsoVisitor<ExecutionContext>() {
109109
@Override
110110
public Toml.Document visitDocument(Toml.Document document, ExecutionContext ctx) {
111-
if (!document.getSourcePath().toString().endsWith("pyproject.toml")) {
111+
String sourcePath = document.getSourcePath().toString();
112+
113+
if (sourcePath.endsWith("uv.lock")) {
114+
PythonDependencyExecutionContextView.view(ctx).getExistingLockContents().put(
115+
PyProjectHelper.correspondingPyprojectPath(sourcePath),
116+
document.printAll());
117+
return document;
118+
}
119+
120+
if (!sourcePath.endsWith("pyproject.toml")) {
112121
return document;
113122
}
114123
Optional<PythonResolutionResult> resolution = document.getMarkers()
@@ -124,7 +133,7 @@ public Toml.Document visitDocument(Toml.Document document, ExecutionContext ctx)
124133
return document;
125134
}
126135

127-
acc.projectsToUpdate.add(document.getSourcePath().toString());
136+
acc.projectsToUpdate.add(sourcePath);
128137
return document;
129138
}
130139
};
@@ -142,10 +151,9 @@ public Toml.Document visitDocument(Toml.Document document, ExecutionContext ctx)
142151
}
143152

144153
if (sourcePath.endsWith("uv.lock")) {
145-
String pyprojectPath = PyProjectHelper.correspondingPyprojectPath(sourcePath);
146-
String newContent = acc.updatedLockFiles.get(pyprojectPath);
147-
if (newContent != null) {
148-
return PyProjectHelper.reparseToml(document, newContent);
154+
Toml.Document updatedLock = PyProjectHelper.maybeUpdateUvLock(document, ctx);
155+
if (updatedLock != null) {
156+
return updatedLock;
149157
}
150158
}
151159

@@ -155,7 +163,7 @@ public Toml.Document visitDocument(Toml.Document document, ExecutionContext ctx)
155163
}
156164

157165
private Toml.Document addDependencyToPyproject(Toml.Document document, ExecutionContext ctx, Accumulator acc) {
158-
String pep508 = version != null ? packageName + version : packageName;
166+
String pep508 = version != null ? packageName + PyProjectHelper.normalizeVersionConstraint(version) : packageName;
159167

160168
Toml.Document updated = (Toml.Document) new TomlIsoVisitor<ExecutionContext>() {
161169
@Override
@@ -228,7 +236,7 @@ public Toml.Array visitArray(Toml.Array array, ExecutionContext ctx) {
228236
}.visitNonNull(document, ctx);
229237

230238
if (updated != document) {
231-
updated = PyProjectHelper.regenerateLockAndRefreshMarker(updated, acc.updatedLockFiles);
239+
updated = PyProjectHelper.regenerateLockAndRefreshMarker(updated, ctx);
232240
}
233241

234242
return updated;

‎rewrite-python/src/main/java/org/openrewrite/python/ChangeDependency.java‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.openrewrite.ScanningRecipe;
2424
import org.openrewrite.TreeVisitor;
2525
import org.openrewrite.python.internal.PyProjectHelper;
26+
import org.openrewrite.python.internal.PythonDependencyExecutionContextView;
2627
import org.openrewrite.python.marker.PythonResolutionResult;
2728
import org.openrewrite.toml.TomlIsoVisitor;
2829
import org.openrewrite.toml.tree.Toml;
@@ -74,7 +75,6 @@ public String getDescription() {
7475

7576
static class Accumulator {
7677
final Set<String> projectsToUpdate = new HashSet<>();
77-
final Map<String, String> updatedLockFiles = new HashMap<>();
7878
}
7979

8080
@Override
@@ -87,7 +87,16 @@ public TreeVisitor<?, ExecutionContext> getScanner(Accumulator acc) {
8787
return new TomlIsoVisitor<ExecutionContext>() {
8888
@Override
8989
public Toml.Document visitDocument(Toml.Document document, ExecutionContext ctx) {
90-
if (!document.getSourcePath().toString().endsWith("pyproject.toml")) {
90+
String sourcePath = document.getSourcePath().toString();
91+
92+
if (sourcePath.endsWith("uv.lock")) {
93+
PythonDependencyExecutionContextView.view(ctx).getExistingLockContents().put(
94+
PyProjectHelper.correspondingPyprojectPath(sourcePath),
95+
document.printAll());
96+
return document;
97+
}
98+
99+
if (!sourcePath.endsWith("pyproject.toml")) {
91100
return document;
92101
}
93102
Optional<PythonResolutionResult> resolution = document.getMarkers()
@@ -98,7 +107,7 @@ public Toml.Document visitDocument(Toml.Document document, ExecutionContext ctx)
98107

99108
PythonResolutionResult marker = resolution.get();
100109
if (marker.findDependencyInAnyScope(oldPackageName) != null) {
101-
acc.projectsToUpdate.add(document.getSourcePath().toString());
110+
acc.projectsToUpdate.add(sourcePath);
102111
}
103112
return document;
104113
}
@@ -117,10 +126,9 @@ public Toml.Document visitDocument(Toml.Document document, ExecutionContext ctx)
117126
}
118127

119128
if (sourcePath.endsWith("uv.lock")) {
120-
String pyprojectPath = PyProjectHelper.correspondingPyprojectPath(sourcePath);
121-
String newContent = acc.updatedLockFiles.get(pyprojectPath);
122-
if (newContent != null) {
123-
return PyProjectHelper.reparseToml(document, newContent);
129+
Toml.Document updatedLock = PyProjectHelper.maybeUpdateUvLock(document, ctx);
130+
if (updatedLock != null) {
131+
return updatedLock;
124132
}
125133
}
126134

@@ -160,7 +168,7 @@ public Toml.Literal visitLiteral(Toml.Literal literal, ExecutionContext ctx) {
160168
sb.append('[').append(extras).append(']');
161169
}
162170
if (newVersion != null) {
163-
sb.append(newVersion);
171+
sb.append(PyProjectHelper.normalizeVersionConstraint(newVersion));
164172
} else {
165173
// Preserve the original version constraint
166174
String originalVersion = extractVersionConstraint(spec, depName);
@@ -178,7 +186,7 @@ public Toml.Literal visitLiteral(Toml.Literal literal, ExecutionContext ctx) {
178186
}.visitNonNull(document, ctx);
179187

180188
if (updated != document) {
181-
updated = PyProjectHelper.regenerateLockAndRefreshMarker(updated, acc.updatedLockFiles);
189+
updated = PyProjectHelper.regenerateLockAndRefreshMarker(updated, ctx);
182190
}
183191

184192
return updated;

‎rewrite-python/src/main/java/org/openrewrite/python/RemoveDependency.java‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.jspecify.annotations.Nullable;
2121
import org.openrewrite.*;
2222
import org.openrewrite.python.internal.PyProjectHelper;
23+
import org.openrewrite.python.internal.PythonDependencyExecutionContextView;
2324
import org.openrewrite.python.marker.PythonResolutionResult;
2425
import org.openrewrite.toml.TomlIsoVisitor;
2526
import org.openrewrite.toml.tree.Space;
@@ -84,7 +85,6 @@ public String getDescription() {
8485

8586
static class Accumulator {
8687
final Set<String> projectsToUpdate = new HashSet<>();
87-
final Map<String, String> updatedLockFiles = new HashMap<>();
8888
}
8989

9090
@Override
@@ -97,7 +97,16 @@ public TreeVisitor<?, ExecutionContext> getScanner(Accumulator acc) {
9797
return new TomlIsoVisitor<ExecutionContext>() {
9898
@Override
9999
public Toml.Document visitDocument(Toml.Document document, ExecutionContext ctx) {
100-
if (!document.getSourcePath().toString().endsWith("pyproject.toml")) {
100+
String sourcePath = document.getSourcePath().toString();
101+
102+
if (sourcePath.endsWith("uv.lock")) {
103+
PythonDependencyExecutionContextView.view(ctx).getExistingLockContents().put(
104+
PyProjectHelper.correspondingPyprojectPath(sourcePath),
105+
document.printAll());
106+
return document;
107+
}
108+
109+
if (!sourcePath.endsWith("pyproject.toml")) {
101110
return document;
102111
}
103112
Optional<PythonResolutionResult> resolution = document.getMarkers()
@@ -113,7 +122,7 @@ public Toml.Document visitDocument(Toml.Document document, ExecutionContext ctx)
113122
return document;
114123
}
115124

116-
acc.projectsToUpdate.add(document.getSourcePath().toString());
125+
acc.projectsToUpdate.add(sourcePath);
117126
return document;
118127
}
119128
};
@@ -131,10 +140,9 @@ public Toml.Document visitDocument(Toml.Document document, ExecutionContext ctx)
131140
}
132141

133142
if (sourcePath.endsWith("uv.lock")) {
134-
String pyprojectPath = PyProjectHelper.correspondingPyprojectPath(sourcePath);
135-
String newContent = acc.updatedLockFiles.get(pyprojectPath);
136-
if (newContent != null) {
137-
return PyProjectHelper.reparseToml(document, newContent);
143+
Toml.Document updatedLock = PyProjectHelper.maybeUpdateUvLock(document, ctx);
144+
if (updatedLock != null) {
145+
return updatedLock;
138146
}
139147
}
140148

@@ -200,7 +208,7 @@ public Toml.Array visitArray(Toml.Array array, ExecutionContext ctx) {
200208
}.visitNonNull(document, ctx);
201209

202210
if (updated != document) {
203-
updated = PyProjectHelper.regenerateLockAndRefreshMarker(updated, acc.updatedLockFiles);
211+
updated = PyProjectHelper.regenerateLockAndRefreshMarker(updated, ctx);
204212
}
205213

206214
return updated;

‎rewrite-python/src/main/java/org/openrewrite/python/UpgradeDependencyVersion.java‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.jspecify.annotations.Nullable;
2121
import org.openrewrite.*;
2222
import org.openrewrite.python.internal.PyProjectHelper;
23+
import org.openrewrite.python.internal.PythonDependencyExecutionContextView;
2324
import org.openrewrite.python.marker.PythonResolutionResult;
2425
import org.openrewrite.toml.TomlIsoVisitor;
2526
import org.openrewrite.toml.tree.Toml;
@@ -88,7 +89,6 @@ public String getDescription() {
8889

8990
static class Accumulator {
9091
final Set<String> projectsToUpdate = new HashSet<>();
91-
final Map<String, String> updatedLockFiles = new HashMap<>();
9292
}
9393

9494
@Override
@@ -101,7 +101,16 @@ public TreeVisitor<?, ExecutionContext> getScanner(Accumulator acc) {
101101
return new TomlIsoVisitor<ExecutionContext>() {
102102
@Override
103103
public Toml.Document visitDocument(Toml.Document document, ExecutionContext ctx) {
104-
if (!document.getSourcePath().toString().endsWith("pyproject.toml")) {
104+
String sourcePath = document.getSourcePath().toString();
105+
106+
if (sourcePath.endsWith("uv.lock")) {
107+
PythonDependencyExecutionContextView.view(ctx).getExistingLockContents().put(
108+
PyProjectHelper.correspondingPyprojectPath(sourcePath),
109+
document.printAll());
110+
return document;
111+
}
112+
113+
if (!sourcePath.endsWith("pyproject.toml")) {
105114
return document;
106115
}
107116
Optional<PythonResolutionResult> resolution = document.getMarkers()
@@ -120,11 +129,11 @@ public Toml.Document visitDocument(Toml.Document document, ExecutionContext ctx)
120129
}
121130

122131
// Skip if the version constraint already matches
123-
if (newVersion.equals(dep.getVersionConstraint())) {
132+
if (PyProjectHelper.normalizeVersionConstraint(newVersion).equals(dep.getVersionConstraint())) {
124133
return document;
125134
}
126135

127-
acc.projectsToUpdate.add(document.getSourcePath().toString());
136+
acc.projectsToUpdate.add(sourcePath);
128137
return document;
129138
}
130139
};
@@ -142,10 +151,9 @@ public Toml.Document visitDocument(Toml.Document document, ExecutionContext ctx)
142151
}
143152

144153
if (sourcePath.endsWith("uv.lock")) {
145-
String pyprojectPath = PyProjectHelper.correspondingPyprojectPath(sourcePath);
146-
String newContent = acc.updatedLockFiles.get(pyprojectPath);
147-
if (newContent != null) {
148-
return PyProjectHelper.reparseToml(document, newContent);
154+
Toml.Document updatedLock = PyProjectHelper.maybeUpdateUvLock(document, ctx);
155+
if (updatedLock != null) {
156+
return updatedLock;
149157
}
150158
}
151159

@@ -208,7 +216,7 @@ private String buildNewSpec(String oldSpec, String depName) {
208216
if (extras != null) {
209217
sb.append('[').append(extras).append(']');
210218
}
211-
sb.append(newVersion);
219+
sb.append(PyProjectHelper.normalizeVersionConstraint(newVersion));
212220
if (marker != null) {
213221
sb.append("; ").append(marker);
214222
}
@@ -217,7 +225,7 @@ private String buildNewSpec(String oldSpec, String depName) {
217225
}.visitNonNull(document, ctx);
218226

219227
if (updated != document) {
220-
updated = PyProjectHelper.regenerateLockAndRefreshMarker(updated, acc.updatedLockFiles);
228+
updated = PyProjectHelper.regenerateLockAndRefreshMarker(updated, ctx);
221229
}
222230

223231
return updated;

0 commit comments

Comments
 (0)
⚡