Also add Unicode support to Proto2 parser

Author: knutwannheden

rewrite-protobuf/src/main/java/org/openrewrite/protobuf/internal/ProtoParserVisitor.java

@@ -44,7 +44,14 @@ public class ProtoParserVisitor extends Protobuf2ParserBaseVisitor<Proto> {
     private final Charset charset;
     private final boolean charsetBomMarked;
 
+    /**
+     * Track position within the file by character (UTF-16 code units)
+     */
     private int cursor = 0;
+    /**
+     * Track parsing position within the file by Unicode code point
+     */
+    private int codePointCursor = 0;
 
     public ProtoParserVisitor(Path path, @Nullable FileAttributes fileAttributes, String source, Charset charset, boolean charsetBomMarked) {
         this.path = path;
@@ -429,12 +436,12 @@ private Space prefix(@Nullable TerminalNode terminalNode) {
 
     private Space prefix(Token token) {
         int start = token.getStartIndex();
-        if (start < cursor) {
+        if (start < codePointCursor) {
             return Space.EMPTY;
         }
-        String prefix = source.substring(cursor, start);
-        cursor = start;
-        return Space.format(prefix);
+        int oldCursor = cursor;
+        advanceCursor(start);
+        return Space.format(source.substring(oldCursor, cursor));
     }
 
     private <C extends ParserRuleContext, T> @Nullable T convert(C ctx, BiFunction<C, Space, T> conversion) {
@@ -445,15 +452,15 @@ private Space prefix(Token token) {
 
         T t = conversion.apply(ctx, prefix(ctx));
         if (ctx.getStop() != null) {
-            cursor = ctx.getStop().getStopIndex() + (Character.isWhitespace(source.charAt(ctx.getStop().getStopIndex())) ? 0 : 1);
+            advanceCursor(ctx.getStop().getStopIndex() + (Character.isWhitespace(source.charAt(ctx.getStop().getStopIndex())) ? 0 : 1));
         }
 
         return t;
     }
 
     private <T> T convert(TerminalNode node, BiFunction<TerminalNode, Space, T> conversion) {
         T t = conversion.apply(node, prefix(node));
-        cursor = node.getSymbol().getStopIndex() + 1;
+        advanceCursor(node.getSymbol().getStopIndex() + 1);
         return t;
     }
 
@@ -469,7 +476,9 @@ private Space sourceBefore(String untilDelim) {
         }
 
         String prefix = source.substring(cursor, delimIndex);
-        cursor += prefix.length() + untilDelim.length(); // advance past the delimiter
+        int codePointsInPrefix = prefix.codePointCount(0, prefix.length());
+        // All Protobuf delimiters are ASCII, so length == code point count
+        advanceCursor(codePointCursor + codePointsInPrefix + untilDelim.length());
         return Space.format(prefix);
     }
 
@@ -515,4 +524,17 @@ private int positionOfNext(String untilDelim, @Nullable Character stop) {
 
         return delimIndex > source.length() - untilDelim.length() ? -1 : delimIndex;
     }
+
+    /**
+     * Advance both the cursor and the code point cursor
+     */
+    @SuppressWarnings("UnusedReturnValue")
+    private int advanceCursor(int newCodePointIndex) {
+        if (newCodePointIndex <= codePointCursor) {
+            return cursor;
+        }
+        cursor = source.offsetByCodePoints(cursor, newCodePointIndex - codePointCursor);
+        codePointCursor = newCodePointIndex;
+        return cursor;
+    }
 }

rewrite-protobuf/src/test/java/org/openrewrite/protobuf/ProtoParserTest.java

@@ -23,11 +23,43 @@
 import java.util.List;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.openrewrite.protobuf.Assertions.proto;
 
 class ProtoParserTest implements RewriteTest {
     @Test
     void noNullsForProto3Files() {
         List<SourceFile> sources = ProtoParser.builder().build().parse("syntax = \"proto3\";").toList();
         assertThat(sources).singleElement().isInstanceOf(PlainText.class);
     }
+
+    @Test
+    void unicodeInComments() {
+        String protoSource = """
+            syntax = "proto2";
+
+            // 👇 Problem below
+            message Person {
+              required string name /*  👆*/=/* 👆*/ 1; /*👆*/
+              required string emoji = 2 [default = "👇"];
+              // 👆 Problem above
+            }
+            """;
+        List<SourceFile> sources = ProtoParser.builder().build().parse(protoSource).toList();
+        assertThat(sources).hasSize(1);
+        assertThat(sources.get(0).printAll()).isEqualTo(protoSource);
+    }
+
+    @Test
+    void moreUnicodeInComments() {
+        rewriteRun(
+          proto(
+            """
+              syntax = 'proto2';
+              service SearchService {
+                /*👆👆*/ rpc /*👆👆*/ Search /*👆👆*/(/*👆👆*/ SearchRequest ) returns ( SearchResponse );
+              }
+              """
+          )
+        );
+    }
 }