Partially collapse blank lines between items (#3274)

Author: casey

src/analyzer.rs

@@ -98,6 +98,7 @@ impl<'run, 'src> Analyzer<'run, 'src> {
               )?);
             }
           }
+          Item::Newline => {}
           Item::Recipe(recipe) => {
             if recipe.enabled() {
               Self::analyze_recipe(recipe)?;

src/ast.rs

@@ -14,18 +14,19 @@ pub(crate) struct Ast<'src> {
 
 impl ColorDisplay for Ast<'_> {
   fn fmt(&self, f: &mut Formatter, color: Color) -> fmt::Result {
-    let mut iter = self.items.iter().peekable();
-
-    while let Some(item) = iter.next() {
-      writeln!(f, "{}", item.color_display(color))?;
+    let mut newline = false;
+    for item in &self.items {
+      if matches!(item, Item::Newline) {
+        newline = true;
+        continue;
+      }
 
-      if let Some(next_item) = iter.peek() {
-        if matches!(item, Item::Recipe(_))
-          || mem::discriminant(item) != mem::discriminant(next_item)
-        {
-          writeln!(f)?;
-        }
+      if newline {
+        writeln!(f)?;
+        newline = false;
       }
+
+      writeln!(f, "{}", item.color_display(color))?;
     }
 
     Ok(())

src/item.rs

@@ -21,6 +21,7 @@ pub(crate) enum Item<'src> {
     private: bool,
     relative: Option<StringLiteral<'src>>,
   },
+  Newline,
   Recipe(UnresolvedRecipe<'src>),
   Set(Set<'src>),
   Unexport {
@@ -85,6 +86,7 @@ impl ColorDisplay for Item<'_> {
 
         Ok(())
       }
+      Self::Newline => Ok(()),
       Self::Recipe(recipe) => write!(f, "{}", recipe.color_display(color)),
       Self::Set(set) => write!(f, "{set}"),
       Self::Unexport { name } => write!(f, "unexport {name}"),

src/lexer.rs

@@ -1,5 +1,7 @@
 use {super::*, CompileErrorKind::*, TokenKind::*};
 
+static DEDENT_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[ \t\n]*\n[^ \t\n]").unwrap());
+
 /// Just language lexer
 ///
 /// The lexer proceeds character-by-character, as opposed to using regular
@@ -391,6 +393,12 @@ impl<'src> Lexer<'src> {
 
     match indentation {
       Blank => {
+        if DEDENT_RE.is_match(self.rest()) {
+          while self.indented() {
+            self.lex_dedent();
+          }
+        }
+
         if !whitespace.is_empty() {
           while self.next_is_whitespace() {
             self.advance()?;
@@ -1472,8 +1480,8 @@ mod tests {
       Indent:"    ",
       Text:"a",
       Eol,
-      Eol,
       Dedent,
+      Eol,
       Identifier:"b",
       Colon,
       Eol,
@@ -1520,8 +1528,8 @@ mod tests {
       Indent,
       Text:"@mv a b",
       Eol,
-      Eol,
       Dedent,
+      Eol,
       Identifier:"a",
       Colon,
       Eol,
@@ -1531,8 +1539,8 @@ mod tests {
       Whitespace:"  ",
       Text:"@touch a",
       Eol,
-      Eol,
       Dedent,
+      Eol,
       Identifier:"d",
       Colon,
       Whitespace,
@@ -1541,8 +1549,8 @@ mod tests {
       Indent,
       Text:"@rm c",
       Eol,
-      Eol,
       Dedent,
+      Eol,
       Identifier:"c",
       Colon,
       Whitespace,
@@ -1820,6 +1828,34 @@ mod tests {
     )
   }
 
+  test! {
+    name: tokenize_continued_indent,
+    text: "
+      hello:
+        foo
+
+        bar
+
+      baz
+    ",
+    tokens: (
+      Identifier:"hello",
+      Colon,
+      Eol,
+      Indent,
+      Text:"foo",
+      Eol,
+      Eol,
+      Whitespace:"  ",
+      Text:"bar",
+      Eol,
+      Dedent,
+      Eol,
+      Identifier:"baz",
+      Eol,
+    ),
+  }
+
   test! {
     name: tokenize_empty_lines,
     text: "
@@ -1856,8 +1892,8 @@ mod tests {
       Whitespace:"  ",
       Text:"dsdf # whatever",
       Eol,
-      Eol,
       Dedent,
+      Eol,
       Comment:"# yolo",
       Eol,
     ),
@@ -1978,8 +2014,8 @@ mod tests {
       Whitespace:"  ",
       Text:"d",
       Eol,
-      Eol,
       Dedent,
+      Eol,
       Comment:"# hello",
       Eol,
       Identifier:"bob",
@@ -2038,8 +2074,8 @@ mod tests {
       Indent,
       Text:"@mv a b",
       Eol,
-      Eol,
       Dedent,
+      Eol,
       Identifier:"a",
       Colon,
       Eol,
@@ -2049,8 +2085,8 @@ mod tests {
       Whitespace:"  ",
       Text:"@touch a",
       Eol,
-      Eol,
       Dedent,
+      Eol,
       Identifier:"d",
       Colon,
       Whitespace,
@@ -2059,8 +2095,8 @@ mod tests {
       Indent,
       Text:"@rm c",
       Eol,
-      Eol,
       Dedent,
+      Eol,
       Identifier:"c",
       Colon,
       Whitespace,
@@ -2227,6 +2263,23 @@ mod tests {
     ),
   }
 
+  test! {
+    name:   tokenize_recipe_after_body,
+    text:   "foo:\n echo FOO\n\nfoo:",
+    tokens: (
+      Identifier: "foo",
+      Colon,
+      Eol,
+      Indent: " ",
+      Text: "echo FOO",
+      Eol,
+      Dedent,
+      Eol,
+      Identifier: "foo",
+      Colon,
+    ),
+  }
+
   error! {
     name:  tokenize_space_then_tab,
     input: "a:

src/node.rs

@@ -10,7 +10,13 @@ pub(crate) trait Node<'src> {
 impl<'src> Node<'src> for Ast<'src> {
   fn tree(&self) -> Tree<'src> {
     Tree::atom("justfile")
-      .extend(self.items.iter().map(Node::tree))
+      .extend(
+        self
+          .items
+          .iter()
+          .filter(|item| !matches!(item, Item::Newline))
+          .map(Node::tree),
+      )
       .extend(self.warnings.iter().map(Node::tree))
   }
 }
@@ -52,6 +58,7 @@ impl<'src> Node<'src> for Item<'src> {
 
         tree
       }
+      Self::Newline => unreachable!(),
       Self::Function(function) => {
         let mut tree = Tree::atom("function");
         tree.push_mut(function.name.lexeme());