fix(graindoc): Preserve indentation in Doc comments during trim (#1119)

chore(stdlib): Regenerate markdown docs for Regex module

Author: phated

compiler/src/parsing/parser_header.re

@@ -30,7 +30,7 @@ let make_doc_comment = (source, loc) => {
   let content =
     String_utils.slice(~first=3, ~last=-2, source)
     |> String_utils.deasterisk_each_line
-    |> String_utils.trim_each_line;
+    |> String_utils.trim_each_line(~style=String_utils.KeepIndent);
   Doc({cmt_content: content, cmt_source: source, cmt_loc: loc});
 };
 

compiler/src/utils/string_utils.re

@@ -13,13 +13,6 @@ let starts_with = (string, prefix) => {
   };
 };
 
-let trim_each_line = str => {
-  str
-  |> Str.split(Str.regexp("\\(\r\n\\|\n\\)"))
-  |> List.map(String.trim)
-  |> String.concat("\n");
-};
-
 let deasterisk_each_line = str => {
   Str.global_replace(Str.regexp("^[ \t]*\\*"), "", str);
 };
@@ -51,6 +44,56 @@ let slice = (~first=?, ~last=?, string) => {
   };
 };
 
+type trim =
+  | KeepIndent
+  | FullTrim;
+
+let get_common_indentation = lines => {
+  let min_whitespace_length =
+    List.fold_left(
+      (min_whitespace_length, line) => {
+        let non_empty_line =
+          Str.string_match(Str.regexp("^\\([ \t]*\\)[^ \t]"), line, 0);
+        if (non_empty_line) {
+          let whitespace = Str.matched_group(1, line);
+          let whitespace_length = String.length(whitespace);
+          switch (min_whitespace_length) {
+          | None => Some(whitespace_length)
+          | Some(min_whitespace_length) =>
+            Some(min(min_whitespace_length, whitespace_length))
+          };
+        } else {
+          min_whitespace_length;
+        };
+      },
+      None,
+      lines,
+    );
+
+  Option.value(~default=0, min_whitespace_length);
+};
+
+let trim_each_line = (~style=FullTrim, str) => {
+  let lines = str |> Str.split(Str.regexp("\\(\r\n\\|\n\\)"));
+
+  let min_whitespace_length =
+    switch (style) {
+    | KeepIndent => get_common_indentation(lines)
+    | FullTrim => 0
+    };
+
+  let trim_style = line => {
+    switch (style) {
+    | KeepIndent =>
+      let line = slice(~first=min_whitespace_length, line);
+      Str.global_replace(Str.regexp("[ \t]+$"), "", line);
+    | FullTrim => String.trim(line)
+    };
+  };
+
+  lines |> List.map(trim_style) |> String.concat("\n");
+};
+
 /** TODO(#436): Re-enable these when we can include in the Windows build
 let%expect_test "empty string" = {
   print_endline(slice(""));

compiler/test/suites/comments.re

@@ -86,13 +86,28 @@ describe("comments", ({test}) => {
     },
   );
   assertParse(
-    "comment_parse_doc_multiline_trim",
-    "/** Test\n    Weird indent\n  Normal indent */\"foo\"",
+    "comment_parse_doc_multiline_trim_all_same_indent",
+    "/**\n  Test\n  Weird indent\n  Normal indent */\"foo\"",
     {
       statements: [str("foo")],
       comments: [
         Parsetree.Doc({
           cmt_content: "Test\nWeird indent\nNormal indent",
+          cmt_source: "/**\n  Test\n  Weird indent\n  Normal indent */",
+          cmt_loc: Location.dummy_loc,
+        }),
+      ],
+      prog_loc: Location.dummy_loc,
+    },
+  );
+  assertParse(
+    "comment_parse_doc_multiline_trim_keeps_differnt_indent",
+    "/** Test\n    Weird indent\n  Normal indent */\"foo\"",
+    {
+      statements: [str("foo")],
+      comments: [
+        Parsetree.Doc({
+          cmt_content: "Test\n   Weird indent\n Normal indent",
           cmt_source: "/** Test\n    Weird indent\n  Normal indent */",
           cmt_loc: Location.dummy_loc,
         }),
@@ -101,14 +116,15 @@ describe("comments", ({test}) => {
     },
   );
   assertParse(
-    "comment_parse_doc_multiline_trim2",
-    "/** Test\r\n    Weird indent\r\n  Normal indent */\"foo\"",
+    "comment_parse_doc_multiline_trim_normalizes_tabs",
+    // Note: There are explicit tab characters in this string to test them
+    "/**\n		Test\r\n	 Weird indent\r\n  Normal indent */\"foo\"",
     {
       statements: [str("foo")],
       comments: [
         Parsetree.Doc({
           cmt_content: "Test\nWeird indent\nNormal indent",
-          cmt_source: "/** Test\r\n    Weird indent\r\n  Normal indent */",
+          cmt_source: "/**\n		Test\r\n	 Weird indent\r\n  Normal indent */",
           cmt_loc: Location.dummy_loc,
         }),
       ],
@@ -152,14 +168,29 @@ describe("comments", ({test}) => {
       statements: [str("foo")],
       comments: [
         Parsetree.Doc({
-          cmt_content: "Test\nno space before\nspace before\ntab before\nno space after",
+          cmt_content: " Test\n no space before\n space before\n tab before\nno space after",
           cmt_source: "/** Test\n* no space before\n * space before\n  * tab before\n *no space after */",
           cmt_loc: Location.dummy_loc,
         }),
       ],
       prog_loc: Location.dummy_loc,
     },
   );
+  assertParse(
+    "comment_parse_doc_deasterisk2",
+    "/** Test\n* no space before\n * space before\n  * tab before\n * trailing space after */\"foo\"",
+    {
+      statements: [str("foo")],
+      comments: [
+        Parsetree.Doc({
+          cmt_content: "Test\nno space before\nspace before\ntab before\ntrailing space after",
+          cmt_source: "/** Test\n* no space before\n * space before\n  * tab before\n * trailing space after */",
+          cmt_loc: Location.dummy_loc,
+        }),
+      ],
+      prog_loc: Location.dummy_loc,
+    },
+  );
   assertCompileError(
     "comment_line_numbers_1",
     "//comment\n//comment\n5 + 5L",

stdlib/regex.md

@@ -62,17 +62,17 @@ The special character sequences are as follows:
 - `(«re»)` - Matches `«re»`, storing the result in a group
 - `(?:«re»)` - Matches `«re»` without storing the result in a group
 - `(?«mode»:«re») - Matches `«re»` with the mode settings specified by `«mode»` using the following syntax:
-- `«mode»i` - The same as `«mode»`, but with case-insensitivity enabled (temporarily not supported until grain-lang/grain#661 is resolved)
-- `«mode»-i` - The same as `«mode»`, but with case-insensitivity disabled (the default)
-- `«mode»m` / `«mode»-s` - The same as `«mode»`, but with multi-line mode enabled
-- `«mode»-m` / `«mode»s` - The same as `«mode»`, but with multi-line mode disabled
-- An empty string, which will not change any mode settings
+  - `«mode»i` - The same as `«mode»`, but with case-insensitivity enabled (temporarily not supported until grain-lang/grain#661 is resolved)
+  - `«mode»-i` - The same as `«mode»`, but with case-insensitivity disabled (the default)
+  - `«mode»m` / `«mode»-s` - The same as `«mode»`, but with multi-line mode enabled
+  - `«mode»-m` / `«mode»s` - The same as `«mode»`, but with multi-line mode disabled
+  - An empty string, which will not change any mode settings
 - `(?«tst»«re1»|«re2»)` - Will match `«re1»` if `«tst»`, otherwise will match `«re2»`. The following options are available for `«tst»`
-- `(«n»)` - Will be true if group `«n»` has a match
-- `(?=«re»)` - Will be true if `«re»` matches the next sequence
-- `(?!«re»)` - Will be true if `«re»` does not match the next sequence
-- `(?<=«re»)` - Will be true if `«re»` matches the preceding sequence
-- `(?<!«re»)` - Will be true if `«re»` does not match the preceding sequence
+  - `(«n»)` - Will be true if group `«n»` has a match
+  - `(?=«re»)` - Will be true if `«re»` matches the next sequence
+  - `(?!«re»)` - Will be true if `«re»` does not match the next sequence
+  - `(?<=«re»)` - Will be true if `«re»` matches the preceding sequence
+  - `(?<!«re»)` - Will be true if `«re»` does not match the preceding sequence
 - `(?«tst»«re»)` - Equivalent to `(?«tst»«re»|)`
 - Finally, basic classes (defined below) can also appear outside of character ranges.