Replace line-based directive interleaving with sentinel-based section assembly

The previous line-number-based approach stored absolute line positions in
DirectiveLine.lineNumber and required all branches to have identical line
counts. If a recipe added or removed lines within a branch, the printer
produced corrupted output.

Ghost comments (//DIRECTIVE:N) are now emitted in clean source in place of
directive lines. These survive Roslyn parsing as whitespace text and act as
positional sentinels. A DirectiveBoundaryInjector scans the parsed tree and
attaches DirectiveBoundaryMarker to nodes adjacent to directive boundaries,
giving recipes structural access to boundary positions. The printer splits
each branch's output by the ghost comment pattern into sections, then
assembles sections from the appropriate branch — allowing different branches
to have different section sizes after recipe modifications.

Author: jkschneider

rewrite-csharp/csharp/OpenRewrite/CSharp/CSharpParser.cs

@@ -42,7 +42,14 @@ private CompilationUnit ParseMulti(string source, string sourcePath,
         SemanticModel? semanticModel, HashSet<string> symbols)
     {
         var directiveLines = PreprocessorSourceTransformer.GetDirectivePositions(source);
-        var permutations = PreprocessorSourceTransformer.GenerateUniquePermutations(source, symbols);
+
+        // Build mapping from line number to directive index for ghost comment emission
+        var directiveLineToIndex = new Dictionary<int, int>();
+        for (int idx = 0; idx < directiveLines.Count; idx++)
+            directiveLineToIndex[directiveLines[idx].LineNumber] = idx;
+
+        var permutations = PreprocessorSourceTransformer.GenerateUniquePermutations(
+            source, symbols, directiveLineToIndex);
         PreprocessorSourceTransformer.ComputeActiveBranchIndices(directiveLines, permutations);
 
         var branches = new List<JRightPadded<CompilationUnit>>();
@@ -66,8 +73,11 @@ private CompilationUnit ParseMulti(string source, string sourcePath,
                 cu = ParseSingle(cleanSource, sourcePath, null);
             }
 
+            // Convert ghost comments in whitespace to DirectiveBoundaryMarker markers
+            cu = (CompilationUnit)DirectiveBoundaryInjector.Inject(cu);
+
             // Add ConditionalBranchMarker to identify this as a branch
-            cu = cu.WithMarkers(cu.Markers.Add(new ConditionalBranchMarker( Guid.NewGuid(), definedSymbols.ToList())));
+            cu = cu.WithMarkers(cu.Markers.Add(new ConditionalBranchMarker(Guid.NewGuid(), definedSymbols.ToList())));
             branches.Add(new JRightPadded<CompilationUnit>(cu, Space.Empty, Markers.Empty));
         }
 

rewrite-csharp/csharp/OpenRewrite/CSharp/CSharpPrinter.cs

@@ -1,3 +1,4 @@
+using System.Text.RegularExpressions;
 using Rewrite.Core;
 using Rewrite.Java;
 
@@ -2464,9 +2465,12 @@ protected void VisitSpace(Space space, PrintOutputCapture<P> p)
 
     #region Preprocessor Directives
 
+    private static readonly Regex GhostCommentPattern = new(@"//DIRECTIVE:(\d+)\r?\n?", RegexOptions.Compiled);
+
     public override J VisitConditionalDirective(ConditionalDirective cd, PrintOutputCapture<P> p)
     {
-        // Print each branch to a separate buffer
+        // 1. Print each branch to a separate buffer.
+        //    Ghost comments (//DIRECTIVE:N) in whitespace appear verbatim in the output.
         var branchOutputs = new string[cd.Branches.Count];
         for (int i = 0; i < cd.Branches.Count; i++)
         {
@@ -2475,58 +2479,84 @@ public override J VisitConditionalDirective(ConditionalDirective cd, PrintOutput
             branchOutputs[i] = capture.ToString();
         }
 
-        // Split each into lines
-        var branchLines = new string[branchOutputs.Length][];
-        for (int i = 0; i < branchOutputs.Length; i++)
+        // 2. Split each branch output by ghost comment sentinels into sections.
+        //    Each branch produces N+1 sections for N directives, with matching directive indices.
+        var branchSections = new List<string>[branchOutputs.Length];
+        var branchTrailingNewlines = new List<bool>[branchOutputs.Length];
+        int[]? directiveOrder = null;
+
+        for (int b = 0; b < branchOutputs.Length; b++)
         {
-            branchLines[i] = branchOutputs[i].Split('\n');
+            var sections = new List<string>();
+            var trailingNewlines = new List<bool>();
+            var dirIndices = new List<int>();
+            var matches = GhostCommentPattern.Matches(branchOutputs[b]);
+
+            int lastEnd = 0;
+            foreach (Match m in matches)
+            {
+                sections.Add(branchOutputs[b][lastEnd..m.Index]);
+                dirIndices.Add(int.Parse(m.Groups[1].Value));
+                trailingNewlines.Add(m.Value.EndsWith('\n'));
+                lastEnd = m.Index + m.Length;
+            }
+            sections.Add(branchOutputs[b][lastEnd..]);
+
+            branchSections[b] = sections;
+            branchTrailingNewlines[b] = trailingNewlines;
+            directiveOrder ??= dirIndices.ToArray();
         }
 
-        // Build directive line lookup
-        var directiveLookup = new Dictionary<int, DirectiveLine>();
-        foreach (var dl in cd.DirectiveLines)
+        // If no ghost comments found (shouldn't happen), fall back to primary branch output
+        if (directiveOrder == null || directiveOrder.Length == 0)
         {
-            directiveLookup[dl.LineNumber] = dl;
+            p.Append(branchOutputs[0]);
+            return cd;
         }
 
-        // Line-level interleaving using stack-based algorithm
+        // 3. Assemble output by interleaving sections with directive text.
+        //    Stack tracks the active branch index (starts with primary branch).
         var stack = new Stack<int>();
-        stack.Push(0); // start with primary branch
-        int totalLines = branchLines[0].Length;
+        stack.Push(0);
 
-        for (int lineNum = 0; lineNum < totalLines; lineNum++)
+        // Section before the first directive — always from primary branch
+        p.Append(branchSections[0][0]);
+
+        for (int d = 0; d < directiveOrder.Length; d++)
         {
-            if (lineNum > 0) p.Append("\n");
+            int directiveIndex = directiveOrder[d];
+            var directive = cd.DirectiveLines[directiveIndex];
 
-            if (directiveLookup.TryGetValue(lineNum, out var directive))
-            {
-                // Emit directive text
-                p.Append(directive.Text);
+            // Emit original directive text (e.g., "#if DEBUG")
+            p.Append(directive.Text);
 
-                switch (directive.Kind)
-                {
-                    case PreprocessorDirectiveKind.If:
-                        stack.Push(directive.ActiveBranchIndex);
-                        break;
-                    case PreprocessorDirectiveKind.Elif:
-                    case PreprocessorDirectiveKind.Else:
-                        stack.Pop();
-                        stack.Push(directive.ActiveBranchIndex);
-                        break;
-                    case PreprocessorDirectiveKind.Endif:
-                        stack.Pop();
-                        break;
-                }
+            // Restore the newline that the ghost comment occupied
+            if (branchTrailingNewlines[0][d])
+                p.Append('\n');
+
+            // Update the active branch stack
+            switch (directive.Kind)
+            {
+                case PreprocessorDirectiveKind.If:
+                    stack.Push(directive.ActiveBranchIndex);
+                    break;
+                case PreprocessorDirectiveKind.Elif:
+                case PreprocessorDirectiveKind.Else:
+                    stack.Pop();
+                    stack.Push(directive.ActiveBranchIndex);
+                    break;
+                case PreprocessorDirectiveKind.Endif:
+                    stack.Pop();
+                    break;
             }
-            else
+
+            // Emit next section from the active branch
+            int activeBranch = stack.Peek();
+            int sectionIndex = d + 1;
+            if (activeBranch < branchSections.Length &&
+                sectionIndex < branchSections[activeBranch].Count)
             {
-                // Emit code from active branch
-                int activeBranch = stack.Peek();
-                if (activeBranch >= 0 && activeBranch < branchLines.Length &&
-                    lineNum < branchLines[activeBranch].Length)
-                {
-                    p.Append(branchLines[activeBranch][lineNum]);
-                }
+                p.Append(branchSections[activeBranch][sectionIndex]);
             }
         }
 

rewrite-csharp/csharp/OpenRewrite/CSharp/Cs.cs

@@ -1051,6 +1051,43 @@ public ConditionalBranchMarker RpcReceive(ConditionalBranchMarker before, RpcRec
     public override int GetHashCode() => Id.GetHashCode();
 }
 
+/// <summary>
+/// Marks a tree node adjacent to one or more conditional preprocessor directive boundaries.
+/// Each directive index references a position in the <see cref="ConditionalDirective.DirectiveLines"/> list.
+/// </summary>
+public sealed class DirectiveBoundaryMarker(
+    Guid id,
+    IList<int> directiveIndices
+) : Marker, IRpcCodec<DirectiveBoundaryMarker>, IEquatable<DirectiveBoundaryMarker>
+{
+    public Guid Id { get; } = id;
+    public IList<int> DirectiveIndices { get; } = directiveIndices;
+
+    public DirectiveBoundaryMarker WithId(Guid id) =>
+        id == Id ? this : new(id, DirectiveIndices);
+    public DirectiveBoundaryMarker WithDirectiveIndices(IList<int> directiveIndices) =>
+        ReferenceEquals(directiveIndices, DirectiveIndices) ? this : new(Id, directiveIndices);
+
+    public void RpcSend(DirectiveBoundaryMarker after, RpcSendQueue q)
+    {
+        q.GetAndSend(after, m => m.Id);
+        q.GetAndSendList(after, m => m.DirectiveIndices, i => i.ToString(), i =>
+            q.GetAndSend(i, x => x.ToString()));
+    }
+
+    public DirectiveBoundaryMarker RpcReceive(DirectiveBoundaryMarker before, RpcReceiveQueue q)
+    {
+        return before
+            .WithId(q.ReceiveAndGet<Guid, string>(before.Id, Guid.Parse))
+            .WithDirectiveIndices(q.ReceiveList(before.DirectiveIndices, idx =>
+                q.ReceiveAndGet<int, string>(idx, int.Parse))!);
+    }
+
+    public bool Equals(DirectiveBoundaryMarker? other) => other is not null && Id == other.Id;
+    public override bool Equals(object? obj) => Equals(obj as DirectiveBoundaryMarker);
+    public override int GetHashCode() => Id.GetHashCode();
+}
+
 /// <summary>
 /// A #pragma warning disable/restore directive.
 /// </summary>

rewrite-csharp/csharp/OpenRewrite/CSharp/DirectiveBoundaryInjector.cs

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2024 the original author or authors.
+ * <p>
+ * Licensed under the Moderne Source Available License (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * https://docs.moderne.io/licensing/moderne-source-available-license
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System.Text.RegularExpressions;
+using Rewrite.Core;
+using Rewrite.Java;
+using static Rewrite.Java.J;
+
+namespace Rewrite.CSharp;
+
+/// <summary>
+/// Scans a parsed tree for ghost comments (//DIRECTIVE:N) embedded in Space whitespace strings
+/// and attaches <see cref="DirectiveBoundaryMarker"/> to the owning nodes.
+/// <para>
+/// Ghost comments are emitted by <see cref="PreprocessorSourceTransformer.Transform"/> in place
+/// of directive lines (e.g. #if, #else, #endif). Because the C# parser does not yet extract
+/// comments from trivia, these appear as raw text in Space.Whitespace rather than as TextComment
+/// entries in Space.Comments.
+/// </para>
+/// <para>
+/// Ghost comments are intentionally left in the whitespace strings so the printer can use them
+/// as sentinels for section-based assembly. The markers provide structured metadata for recipes.
+/// </para>
+/// </summary>
+public partial class DirectiveBoundaryInjector : CSharpVisitor<int>
+{
+    [GeneratedRegex(@"//DIRECTIVE:(\d+)")]
+    private static partial Regex GhostPattern();
+
+    /// <summary>
+    /// Inject directive boundary markers into the given tree by scanning for ghost comments.
+    /// Ghost comments remain in the whitespace for printer use as sentinels.
+    /// </summary>
+    public static J Inject(J tree)
+    {
+        return new DirectiveBoundaryInjector().VisitNonNull(tree, 0);
+    }
+
+    public override J? PostVisit(J tree, int p)
+    {
+        return AddMarkerIfNeeded(tree, tree.Prefix.Whitespace);
+    }
+
+    public override J VisitBlock(Block block, int p)
+    {
+        block = (Block)base.VisitBlock(block, p);
+
+        var indices = FindDirectiveIndices(block.End.Whitespace);
+        if (indices.Count > 0)
+        {
+            var marker = new DirectiveBoundaryMarker(Guid.NewGuid(), indices);
+            block = block.WithMarkers(block.Markers.Add(marker));
+        }
+
+        return block;
+    }
+
+    public override J VisitCompilationUnit(CompilationUnit compilationUnit, int p)
+    {
+        compilationUnit = (CompilationUnit)base.VisitCompilationUnit(compilationUnit, p);
+
+        var indices = FindDirectiveIndices(compilationUnit.Eof.Whitespace);
+        if (indices.Count > 0)
+        {
+            var marker = new DirectiveBoundaryMarker(Guid.NewGuid(), indices);
+            compilationUnit = compilationUnit.WithMarkers(compilationUnit.Markers.Add(marker));
+        }
+
+        return compilationUnit;
+    }
+
+    private J AddMarkerIfNeeded(J node, string whitespace)
+    {
+        var indices = FindDirectiveIndices(whitespace);
+        if (indices.Count == 0)
+            return node;
+
+        var marker = new DirectiveBoundaryMarker(Guid.NewGuid(), indices);
+        var newMarkers = node.Markers.Add(marker);
+        return SetMarkers(node, newMarkers);
+    }
+
+    private static List<int> FindDirectiveIndices(string whitespace)
+    {
+        var indices = new List<int>();
+        var matches = GhostPattern().Matches(whitespace);
+        foreach (Match match in matches)
+        {
+            indices.Add(int.Parse(match.Groups[1].Value));
+        }
+        return indices;
+    }
+
+    /// <summary>
+    /// Uses reflection to call WithMarkers on any concrete J type,
+    /// following the pattern established by <see cref="SearchResult.Found{T}"/>.
+    /// </summary>
+    private static J SetMarkers(J node, Markers markers)
+    {
+        var withMarkers = node.GetType().GetMethod("WithMarkers", [typeof(Markers)]);
+        return withMarkers != null ? (J)withMarkers.Invoke(node, [markers])! : node;
+    }
+}