From 2a1d78b0ef40e9da593b7ce2d41dc51f754ab59e Mon Sep 17 00:00:00 2001 From: bmazzarol Date: Sun, 20 Jul 2025 23:34:43 +0800 Subject: [PATCH 01/10] feat: new parser WIP --- Cutout.Tests/Cutout.Tests.csproj | 37 ++ Cutout.Tests/LexerTests.cs | 407 +++++++++++++++ Cutout.Tests/ParserTests.cs | 462 ++++++++++++++++++ Cutout.Tests/TemplateParserTests.cs | 57 ++- Cutout/Cutout.csproj | 10 +- Cutout/Exceptions/ParseException.cs | 7 +- Cutout/Extensions/TokenExtensions.cs | 25 - Cutout/Extensions/TokenListExtensions.cs | 16 + Cutout/GlobalUsings.cs | 2 + Cutout/Lexer/Lexer.cs | 189 +++++++ Cutout/Lexer/Token.cs | 72 +++ Cutout/Parser/Identifiers.cs | 19 - Cutout/Parser/Parser.cs | 318 ++++++++++++ Cutout/Parser/Syntax.cs | 121 +---- Cutout/Parser/TemplateParser.CallStatement.cs | 77 --- .../TemplateParser.EndBreakPredicate.cs | 36 -- Cutout/Parser/TemplateParser.ForStatements.cs | 60 --- Cutout/Parser/TemplateParser.IParseContext.cs | 15 - Cutout/Parser/TemplateParser.IfStatements.cs | 111 ----- .../Parser/TemplateParser.KeywordStatement.cs | 25 - Cutout/Parser/TemplateParser.VarStatement.cs | 18 - Cutout/Parser/TemplateParser.cs | 338 ------------- Cutout/Renderer/Renderer.cs | 2 +- Cutout/TemplateAttributeParts.cs | 8 +- ...plateSourceGenerator.TemplateMethodImpl.cs | 8 +- Parent.Directory.Packages.props | 1 - 26 files changed, 1579 insertions(+), 862 deletions(-) create mode 100644 Cutout.Tests/LexerTests.cs create mode 100644 Cutout.Tests/ParserTests.cs delete mode 100644 Cutout/Extensions/TokenExtensions.cs create mode 100644 Cutout/Extensions/TokenListExtensions.cs create mode 100644 Cutout/GlobalUsings.cs create mode 100644 Cutout/Lexer/Lexer.cs create mode 100644 Cutout/Lexer/Token.cs delete mode 100644 Cutout/Parser/Identifiers.cs create mode 100644 Cutout/Parser/Parser.cs delete mode 100644 Cutout/Parser/TemplateParser.CallStatement.cs delete mode 100644 Cutout/Parser/TemplateParser.EndBreakPredicate.cs delete mode 100644 Cutout/Parser/TemplateParser.ForStatements.cs delete mode 100644 Cutout/Parser/TemplateParser.IParseContext.cs delete mode 100644 Cutout/Parser/TemplateParser.IfStatements.cs delete mode 100644 Cutout/Parser/TemplateParser.KeywordStatement.cs delete mode 100644 Cutout/Parser/TemplateParser.VarStatement.cs delete mode 100644 Cutout/Parser/TemplateParser.cs diff --git a/Cutout.Tests/Cutout.Tests.csproj b/Cutout.Tests/Cutout.Tests.csproj index f14bc58..0e073ee 100644 --- a/Cutout.Tests/Cutout.Tests.csproj +++ b/Cutout.Tests/Cutout.Tests.csproj @@ -21,4 +21,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Cutout.Tests/LexerTests.cs b/Cutout.Tests/LexerTests.cs new file mode 100644 index 0000000..cca10a3 --- /dev/null +++ b/Cutout.Tests/LexerTests.cs @@ -0,0 +1,407 @@ +namespace Cutout.Tests; + +public class LexerTests +{ + [Fact(DisplayName = "Whitespace can be tokenized")] + public void Case1() + { + const string text = " \t\n\r "; + var tokens = Lexer.Tokenize(text.AsSpan()); + + Assert.Collection( + tokens, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(new CharPosition(1, 1, 0), token.Start); + Assert.Equal(new CharPosition(1, 3, 2), token.End); + Assert.Equal(" \t", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Newline, token.Type); + Assert.Equal(new CharPosition(1, 4, 3), token.Start); + Assert.Equal(new CharPosition(1, 4, 3), token.End); + Assert.Equal("\n", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(new CharPosition(2, 1, 4), token.Start); + Assert.Equal(new CharPosition(2, 3, 6), token.End); + Assert.Equal("\r ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Eof, token.Type); + Assert.Equal(new CharPosition(0, 0, -1), token.Start); + Assert.Equal(new CharPosition(0, 0, -1), token.End); + Assert.Equal(string.Empty, token.ToSpan(text.AsSpan()).ToString()); + } + ); + } + + [Fact(DisplayName = "Code blocks can be tokenized")] + public void Case2() + { + const string text = "{{ code block }}"; + var tokens = Lexer.Tokenize(text.AsSpan()); + Assert.Collection( + tokens, + token => + { + Assert.Equal(TokenType.CodeEnter, token.Type); + Assert.Equal(0, token.Start.Offset); + Assert.Equal(1, token.End.Offset); + Assert.Equal("{{", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(2, token.Start.Offset); + Assert.Equal(2, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(3, token.Start.Offset); + Assert.Equal(6, token.End.Offset); + Assert.Equal("code", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(7, token.Start.Offset); + Assert.Equal(7, token.End.Offset); + + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(8, token.Start.Offset); + Assert.Equal(12, token.End.Offset); + Assert.Equal("block", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(13, token.Start.Offset); + Assert.Equal(13, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.CodeExit, token.Type); + Assert.Equal(14, token.Start.Offset); + Assert.Equal(15, token.End.Offset); + Assert.Equal("}}", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Eof, token.Type); + Assert.Equal(-1, token.Start.Offset); + Assert.Equal(-1, token.End.Offset); + } + ); + } + + [Fact(DisplayName = "Raw text can be tokenized")] + public void Case3() + { + const string text = "This is some raw text"; + var tokens = Lexer.Tokenize(text.AsSpan()); + Assert.Collection( + tokens, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(0, token.Start.Offset); + Assert.Equal(3, token.End.Offset); + Assert.Equal("This", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(4, token.Start.Offset); + Assert.Equal(4, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(5, token.Start.Offset); + Assert.Equal(6, token.End.Offset); + Assert.Equal("is", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(7, token.Start.Offset); + Assert.Equal(7, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(8, token.Start.Offset); + Assert.Equal(11, token.End.Offset); + Assert.Equal("some", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(12, token.Start.Offset); + Assert.Equal(12, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(13, token.Start.Offset); + Assert.Equal(15, token.End.Offset); + Assert.Equal("raw", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(16, token.Start.Offset); + Assert.Equal(16, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(17, token.Start.Offset); + Assert.Equal(20, token.End.Offset); + Assert.Equal("text", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Eof, token.Type); + Assert.Equal(-1, token.Start.Offset); + Assert.Equal(-1, token.End.Offset); + } + ); + } + + [Fact(DisplayName = "Mixed text can be tokenized")] + public void Case4() + { + const string text = "This is {{ code block }}\n some raw text\r\n and {{ more code }}"; + var tokens = Lexer.Tokenize(text.AsSpan()); + Assert.Collection( + tokens, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(0, token.Start.Offset); + Assert.Equal(3, token.End.Offset); + Assert.Equal("This", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(4, token.Start.Offset); + Assert.Equal(4, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(5, token.Start.Offset); + Assert.Equal(6, token.End.Offset); + Assert.Equal("is", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(7, token.Start.Offset); + Assert.Equal(7, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.CodeEnter, token.Type); + Assert.Equal(8, token.Start.Offset); + Assert.Equal(9, token.End.Offset); + Assert.Equal("{{", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(10, token.Start.Offset); + Assert.Equal(10, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(11, token.Start.Offset); + Assert.Equal(14, token.End.Offset); + Assert.Equal("code", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(15, token.Start.Offset); + Assert.Equal(15, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(16, token.Start.Offset); + Assert.Equal(20, token.End.Offset); + Assert.Equal("block", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(21, token.Start.Offset); + Assert.Equal(21, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.CodeExit, token.Type); + Assert.Equal(22, token.Start.Offset); + Assert.Equal(23, token.End.Offset); + Assert.Equal("}}", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Newline, token.Type); + Assert.Equal(24, token.Start.Offset); + Assert.Equal(24, token.End.Offset); + Assert.Equal("\n", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(25, token.Start.Offset); + Assert.Equal(25, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(26, token.Start.Offset); + Assert.Equal(29, token.End.Offset); + Assert.Equal("some", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(30, token.Start.Offset); + Assert.Equal(30, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(31, token.Start.Offset); + Assert.Equal(33, token.End.Offset); + Assert.Equal("raw", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(34, token.Start.Offset); + Assert.Equal(34, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(35, token.Start.Offset); + Assert.Equal(38, token.End.Offset); + Assert.Equal("text", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Newline, token.Type); + Assert.Equal(39, token.Start.Offset); + Assert.Equal(40, token.End.Offset); + Assert.Equal("\r\n", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(41, token.Start.Offset); + Assert.Equal(41, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(42, token.Start.Offset); + Assert.Equal(44, token.End.Offset); + Assert.Equal("and", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(45, token.Start.Offset); + Assert.Equal(45, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.CodeEnter, token.Type); + Assert.Equal(46, token.Start.Offset); + Assert.Equal(47, token.End.Offset); + Assert.Equal("{{", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(48, token.Start.Offset); + Assert.Equal(48, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(49, token.Start.Offset); + Assert.Equal(52, token.End.Offset); + Assert.Equal("more", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(53, token.Start.Offset); + Assert.Equal(53, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(54, token.Start.Offset); + Assert.Equal(57, token.End.Offset); + Assert.Equal("code", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(58, token.Start.Offset); + Assert.Equal(58, token.End.Offset); + Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.CodeExit, token.Type); + Assert.Equal(59, token.Start.Offset); + Assert.Equal(60, token.End.Offset); + Assert.Equal("}}", token.ToSpan(text.AsSpan()).ToString()); + }, + token => + { + Assert.Equal(TokenType.Eof, token.Type); + Assert.Equal(-1, token.Start.Offset); + Assert.Equal(-1, token.End.Offset); + } + ); + } +} diff --git a/Cutout.Tests/ParserTests.cs b/Cutout.Tests/ParserTests.cs new file mode 100644 index 0000000..50be486 --- /dev/null +++ b/Cutout.Tests/ParserTests.cs @@ -0,0 +1,462 @@ +using Cutout.Exceptions; +using Cutout.Extensions; + +namespace Cutout.Tests; + +public class ParserTests +{ + [Fact(DisplayName = "A raw string can be parsed")] + public void Case1() + { + const string template = "raw string"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + var single = Assert.Single(result); + var raw = Assert.IsType(single); + Assert.Collection( + raw.Value, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(token.ToSpan(template), "raw"); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(token.ToSpan(template), "string"); + } + ); + } + + [Fact(DisplayName = "A renderable code block can be parsed")] + public void Case2() + { + const string template = "{{ code }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + var item = Assert.Single(result); + var renderableExpression = Assert.IsType(item); + Assert.Collection( + renderableExpression.Value, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(token.ToSpan(template), "code"); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + } + ); + } + + [Fact(DisplayName = "Raw and renderable code blocks can be parsed")] + public void Case3() + { + const string template = "raw {{ code }} string"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + Assert.Collection( + result, + item => + { + var rawText = Assert.IsType(item); + Assert.Collection( + rawText.Value, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(token.ToSpan(template), "raw"); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + } + ); + }, + item => + { + var renderableExpression = Assert.IsType(item); + Assert.Collection( + renderableExpression.Value, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(token.ToSpan(template), "code"); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + } + ); + }, + item => + { + var rawText = Assert.IsType(item); + Assert.Collection( + rawText.Value, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(token.ToSpan(template), "string"); + } + ); + } + ); + } + + [Fact(DisplayName = "An empty template can be parsed")] + public void Case4() + { + const string template = ""; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + Assert.Empty(result); + } + + [Fact(DisplayName = "Code blocks must have balanced braces")] + public void Case5() + { + const string template = "{{ code"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal("Parse error at end of file: Code exit token not found", exception.Message); + Assert.Equal(string.Empty, exception.Token.ToSpan(template).ToString()); + } + + [Fact(DisplayName = "Code blocks cannot be empty")] + public void Case6() + { + const string template = "{{ }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:3 (Whitespace): Code block is empty (value: ' ')", + exception.Message + ); + Assert.Equal(TokenType.Whitespace, exception.Token.Type); + Assert.Equal(" ", exception.Value); + } + + [Fact(DisplayName = "Nested code blocks are not allowed")] + public void Case7() + { + const string template = "{{ {{ nested }} }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (CodeEnter): Nested code blocks are not allowed (value: '{{')", + exception.Message + ); + Assert.Equal(TokenType.CodeEnter, exception.Token.Type); + Assert.Equal("{{", exception.Value); + } + + [Fact(DisplayName = "A break statement can be parsed")] + public void Case8() + { + const string template = "{{ break }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + var item = Assert.Single(result); + Assert.IsType(item); + } + + [Fact(DisplayName = "A continue statement can be parsed")] + public void Case9() + { + const string template = "{{ continue }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + var item = Assert.Single(result); + Assert.IsType(item); + } + + [Fact(DisplayName = "A return statement can be parsed")] + public void Case10() + { + const string template = "{{ return }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + var item = Assert.Single(result); + Assert.IsType(item); + } + + [Fact(DisplayName = "A var statement can be parsed")] + public void Case11() + { + const string template = "{{ var x = 42 }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + var item = Assert.Single(result); + var varDeclaration = Assert.IsType(item); + Assert.Collection( + varDeclaration.Assignment, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(token.ToSpan(template), "x"); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(token.ToSpan(template), "="); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(token.ToSpan(template), "42"); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + } + ); + } + + [Fact(DisplayName = "A var statement without an expression throws an error")] + public void Case12() + { + const string template = "{{ var }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): Variable declaration requires an expression (value: 'var')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("var", exception.Value); + } + + [Fact(DisplayName = "A call statement can be parsed")] + public void Case13() + { + const string template = "{{ call function(arg1, arg2) }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + var item = Assert.Single(result); + var callStatement = Assert.IsType(item); + Assert.Equal("function", callStatement.Name); + Assert.Collection( + callStatement.Parameters, + param => + { + Assert.Equal("arg1", param); + }, + param => + { + Assert.Equal("arg2", param); + } + ); + } + + [Fact(DisplayName = "A call statement function part throws an error")] + public void Case14() + { + const string template = "{{ call }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): Call statement requires parameters (value: 'call')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("call", exception.Value); + } + + [Fact(DisplayName = "A call statement without parentheses throws an error")] + public void Case15() + { + const string template = "{{ call function }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): Call statement requires a function name and () with optional parameters (value: 'call')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("call", exception.Value); + } + + [Fact(DisplayName = "A call statement with only parentheses throws an error")] + public void Case16() + { + const string template = "{{ call () }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): Call statement requires a function name and () with optional parameters (value: 'call')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("call", exception.Value); + } + + [Fact(DisplayName = "A while statement can be parsed")] + public void Case17() + { + const string template = "{{ while condition }} some code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + var item = Assert.Single(result); + var whileStatement = Assert.IsType(item); + Assert.Collection( + whileStatement.Condition, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(token.ToSpan(template), "condition"); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + } + ); + + var expression = Assert.Single(whileStatement.Expressions); + var rawText = Assert.IsType(expression); + Assert.Collection( + rawText.Value, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(token.ToSpan(template), "some"); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal(token.ToSpan(template), "code"); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(token.ToSpan(template), " "); + } + ); + } + + [Fact(DisplayName = "A while statement without a condition throws an error")] + public void Case18() + { + const string template = "{{ while }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): while statement requires a condition (value: 'while')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("while", exception.Value); + } + + [Fact(DisplayName = "A while statement without an end token throws an error")] + public void Case19() + { + const string template = "{{ while condition }} some code"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at end of file: Unexpected end of file, expected a {{ end }} block", + exception.Message + ); + Assert.Equal(TokenType.Eof, exception.Token.Type); + Assert.Equal(string.Empty, exception.Value); + } + + [Fact(DisplayName = "A for statement can be parsed")] + public void Case20() + { + const string template = "{{ for i = 0; i < items.Length; i++ }} some code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + var item = Assert.Single(result); + var forStatement = Assert.IsType(item); + Assert.Equal(" i = 0; i < items.Length; i++ ", forStatement.Condition.ToSpan(template)); + + var expression = Assert.Single(forStatement.Expressions); + var rawText = Assert.IsType(expression); + Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + } + + [Fact(DisplayName = "A foreach statement can be parsed")] + public void Case21() + { + const string template = "{{ foreach item in items }} some code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + var item = Assert.Single(result); + var foreachStatement = Assert.IsType(item); + Assert.Equal(" item in items ", foreachStatement.Condition.ToSpan(template)); + + var expression = Assert.Single(foreachStatement.Expressions); + var rawText = Assert.IsType(expression); + Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + } +} diff --git a/Cutout.Tests/TemplateParserTests.cs b/Cutout.Tests/TemplateParserTests.cs index 980e954..05f5fa7 100644 --- a/Cutout.Tests/TemplateParserTests.cs +++ b/Cutout.Tests/TemplateParserTests.cs @@ -1,6 +1,5 @@ using Cutout.Exceptions; using Cutout.Parser; -using Scriban.Parsing; namespace Cutout.Tests; @@ -10,7 +9,7 @@ public sealed class TemplateParserTests public void Case1() { const string template = "raw string"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); var item = Assert.Single(result); @@ -22,7 +21,7 @@ public void Case1() public void Case2() { const string template = "{{ code }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); var item = Assert.Single(result); @@ -34,7 +33,7 @@ public void Case2() public void Case3() { const string template = "raw {{ code }} string"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); Assert.Collection( @@ -61,7 +60,7 @@ public void Case3() public void Case4() { const string template = "{{ if condition }} test {{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); var item = Assert.Single(result); @@ -76,7 +75,7 @@ public void Case4() public void Case5() { const string template = "{{ if condition }} test {{ else }} test2 {{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); Assert.Collection( @@ -104,7 +103,7 @@ public void Case6() { const string template = "{{ if condition1 }} test {{ else if condition2 }} test2 {{ else }} test3 {{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); Assert.Collection( @@ -139,7 +138,7 @@ public void Case6() public void Case7() { const string template = "{{ if condition1 }} test {{ else if condition2 }} test2 {{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); Assert.Collection( @@ -167,7 +166,7 @@ public void Case7() public void Case8() { const string template = "{{ if condition1 }} test {{ else if }} test2 {{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); Assert.Equal( "Parse error at 0:36 (CodeExit): else if statement condition not found (value: '}}')", @@ -180,7 +179,7 @@ public void Case9() { const string template = "{{ if condition1 }} test {{ else invalid condition }} test2 {{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); Assert.Equal( "Parse error at 0:33 (Identifier): else statement must be followed by if statement (value: 'invalid')", @@ -192,7 +191,7 @@ public void Case9() public void Case10() { const string template = "{{ if }} test {{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); Assert.Equal( "Parse error at 0:6 (CodeExit): if statement condition not found (value: '}}')", @@ -204,7 +203,7 @@ public void Case10() public void Case11() { const string template = "{{ if condition1 }} test {{ else if condition2 }} test2 "; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); Assert.Equal( "Parse error at 0:49 (Raw): else or end not found (value: ' test2 ')", @@ -216,7 +215,7 @@ public void Case11() public void Case12() { const string template = "{{ if condition1 }}{{ if condition2 }} test {{ end }}{{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); var item = Assert.Single(result); @@ -233,7 +232,7 @@ public void Case12() public void Case13() { const string template = "{{ for condition }} test {{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); var item = Assert.Single(result); @@ -248,7 +247,7 @@ public void Case13() public void Case14() { const string template = "{{ for }} test {{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); Assert.Equal( "Parse error at 0:7 (CodeExit): for statement condition not found (value: '}}')", @@ -260,7 +259,7 @@ public void Case14() public void Case15() { const string template = "{{ foreach condition }} test {{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); var item = Assert.Single(result); @@ -277,7 +276,7 @@ public void Case15() public void Case16() { const string template = "{{ foreach }} test {{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); Assert.Equal( "Parse error at 0:11 (CodeExit): foreach statement condition not found (value: '}}')", @@ -289,7 +288,7 @@ public void Case16() public void Case17() { const string template = "{{ while condition }} test {{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); var item = Assert.Single(result); @@ -306,7 +305,7 @@ public void Case17() public void Case18() { const string template = "{{ while }} test {{ end }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); Assert.Equal( "Parse error at 0:9 (CodeExit): while statement condition not found (value: '}}')", @@ -318,7 +317,7 @@ public void Case18() public void Case19() { const string template = "{{ continue }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); var item = Assert.Single(result); @@ -329,7 +328,7 @@ public void Case19() public void Case20() { const string template = "{{ break }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); var item = Assert.Single(result); @@ -340,7 +339,7 @@ public void Case20() public void Case21() { const string template = "{{ break invalid }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); Assert.Equal( "Parse error at 0:9 (Identifier): Expected only keyword 'break' (value: 'invalid')", @@ -360,7 +359,7 @@ public void Case22() The value is {{i}} {{ end }} """; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); var whileStatement = Assert.IsType(Assert.Single(result)); @@ -396,7 +395,7 @@ The value is {{i}} public void Case23() { const string template = "{{ return }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); var item = Assert.Single(result); @@ -407,7 +406,7 @@ public void Case23() public void Case24() { const string template = "{{ return invalid }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); Assert.Equal( "Parse error at 0:10 (Identifier): Expected only keyword 'return' (value: 'invalid')", @@ -424,7 +423,7 @@ public void Case25() {{continue}} {{return}} """; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); Assert.Collection( @@ -444,7 +443,7 @@ public void Case25() public void Case26() { const string template = "{{ call method(param1, param2) }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); var item = Assert.Single(result); @@ -458,7 +457,7 @@ public void Case26() public void Case27() { const string template = "{{ call method(param1, param2) invalid }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); Assert.Equal( "Parse error at 0:31 (Identifier): Invalid call statement. Expected format: 'MethodName(...)'. (value: 'invalid')", @@ -471,7 +470,7 @@ public void Case28() { const string template = "some text \n some other text \n\n {{ call method(param1, param2) }}"; - var tokens = new Lexer(template).ToArray(); + var tokens = Lexer.Tokenize(template); var result = TemplateParser.Parse(tokens, template); Assert.Collection( diff --git a/Cutout/Cutout.csproj b/Cutout/Cutout.csproj index 2b2ebbf..87fe16a 100644 --- a/Cutout/Cutout.csproj +++ b/Cutout/Cutout.csproj @@ -34,7 +34,15 @@ - + + + + + + + + + diff --git a/Cutout/Exceptions/ParseException.cs b/Cutout/Exceptions/ParseException.cs index 0f8c464..a59e3f8 100644 --- a/Cutout/Exceptions/ParseException.cs +++ b/Cutout/Exceptions/ParseException.cs @@ -1,4 +1,4 @@ -using Scriban.Parsing; +using System.Globalization; namespace Cutout.Exceptions; @@ -16,6 +16,11 @@ internal ParseException(Token token, string value, string message) private static string BuildMessage(Token token, string value, string message) { + if (token.Type is TokenType.Eof) + { + return $"Parse error at end of file: {message}"; + } + var tokenType = token.Type.ToString(); var lineNumber = token.Start.Line; var columnNumber = token.Start.Column; diff --git a/Cutout/Extensions/TokenExtensions.cs b/Cutout/Extensions/TokenExtensions.cs deleted file mode 100644 index 6b38721..0000000 --- a/Cutout/Extensions/TokenExtensions.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Scriban.Parsing; - -namespace Cutout.Extensions; - -internal static class TokenExtensions -{ - internal static ReadOnlySpan ToSpan( - this Token token, - ReadOnlySpan template, - Token? end = null - ) - { - if (token.Type == TokenType.Eof) - { - return []; - } - - var endToken = end ?? token; - - return template.Slice( - token.Start.Offset, - length: Math.Max(endToken.End.Offset - token.Start.Offset + 1, 0) - ); - } -} diff --git a/Cutout/Extensions/TokenListExtensions.cs b/Cutout/Extensions/TokenListExtensions.cs new file mode 100644 index 0000000..1189c7f --- /dev/null +++ b/Cutout/Extensions/TokenListExtensions.cs @@ -0,0 +1,16 @@ +namespace Cutout.Extensions; + +internal static class TokenListExtensions +{ + internal static ReadOnlySpan ToSpan(this TokenList list, in ReadOnlySpan template) + { + if (list.Count == 0) + { + return ReadOnlySpan.Empty; + } + + var start = list[0]; + var end = list[list.Count - 1]; + return start.ToSpan(in template, end); + } +} diff --git a/Cutout/GlobalUsings.cs b/Cutout/GlobalUsings.cs new file mode 100644 index 0000000..0a232fc --- /dev/null +++ b/Cutout/GlobalUsings.cs @@ -0,0 +1,2 @@ +global using SyntaxList = System.Collections.Generic.List; +global using TokenList = System.Collections.Generic.List; diff --git a/Cutout/Lexer/Lexer.cs b/Cutout/Lexer/Lexer.cs new file mode 100644 index 0000000..90da6b8 --- /dev/null +++ b/Cutout/Lexer/Lexer.cs @@ -0,0 +1,189 @@ +namespace Cutout; + +internal static class Lexer +{ + internal static TokenList Tokenize(in ReadOnlySpan text) + { + var tokens = new TokenList(); + var i = 0; + var line = 1; + var column = 1; + var length = text.Length; + + while ( + i < length + && ( + TryProcessNewline(text, ref i, ref line, ref column, length, tokens) + || TryProcessCodeDelimiters(text, ref i, line, ref column, length, tokens) + || TryProcessWhitespace(text, ref i, line, ref column, length, tokens) + || TryProcessRawText(text, ref i, line, ref column, length, tokens) + ) + ) + { // collect tokens + } + + tokens.Add(Token.Eof); + return tokens; + } + + private static bool TryProcessNewline( + ReadOnlySpan text, + ref int i, + ref int line, + ref int column, + int length, + TokenList tokens + ) + { + var c = text[i]; + var currentPosition = new CharPosition(line, column, i); + + switch (c) + { + case '\r' when i + 1 < length && text[i + 1] == '\n': + tokens.Add( + new Token( + currentPosition, + new CharPosition(line, column + 1, i + 1), + TokenType.Newline + ) + ); + i += 2; + line++; + column = 1; + return true; + case '\n': + tokens.Add(new Token(currentPosition, currentPosition, TokenType.Newline)); + i++; + line++; + column = 1; + return true; + default: + return false; + } + } + + private static bool TryProcessCodeDelimiters( + ReadOnlySpan text, + ref int i, + int line, + ref int column, + int length, + TokenList tokens + ) + { + var c = text[i]; + var currentPosition = new CharPosition(line, column, i); + + switch (c) + { + case '{' when i + 1 < length && text[i + 1] == '{': + tokens.Add( + new Token( + currentPosition, + new CharPosition(line, column + 1, i + 1), + TokenType.CodeEnter + ) + ); + i += 2; + column += 2; + return true; + case '}' when i + 1 < length && text[i + 1] == '}': + tokens.Add( + new Token( + currentPosition, + new CharPosition(line, column + 1, i + 1), + TokenType.CodeExit + ) + ); + i += 2; + column += 2; + return true; + default: + return false; + } + } + + private static bool TryProcessWhitespace( + ReadOnlySpan text, + ref int i, + int line, + ref int column, + int length, + TokenList tokens + ) + { + var c = text[i]; + + if ( + !char.IsWhiteSpace(c) + || c == '\n' + || (c == '\r' && i + 1 < length && text[i + 1] == '\n') + ) + { + return false; + } + + var start = i; + var columnStart = column; + + while ( + i < length + && char.IsWhiteSpace(text[i]) + && text[i] != '\n' + && !(text[i] == '\r' && i + 1 < length && text[i + 1] == '\n') + ) + { + i++; + column++; + } + + tokens.Add( + new Token( + new CharPosition(line, columnStart, start), + new CharPosition(line, column - 1, i - 1), + TokenType.Whitespace + ) + ); + + return true; + } + + private static bool TryProcessRawText( + ReadOnlySpan text, + ref int i, + int line, + ref int column, + int length, + TokenList tokens + ) + { + var start = i; + var columnStart = column; + + while ( + i < length + && !( + char.IsWhiteSpace(text[i]) + || (text[i] == '{' && i + 1 < length && text[i + 1] == '{') + || (text[i] == '}' && i + 1 < length && text[i + 1] == '}') + || text[i] == '\n' + || (text[i] == '\r' && i + 1 < length && text[i + 1] == '\n') + ) + ) + { + i++; + column++; + } + + tokens.Add( + new Token( + new CharPosition(line, columnStart, start), + new CharPosition(line, column - 1, i - 1), + TokenType.Raw + ) + ); + + return true; + } +} diff --git a/Cutout/Lexer/Token.cs b/Cutout/Lexer/Token.cs new file mode 100644 index 0000000..67ca53d --- /dev/null +++ b/Cutout/Lexer/Token.cs @@ -0,0 +1,72 @@ +using System.Runtime.InteropServices; + +namespace Cutout; + +internal enum TokenType : byte +{ + /// + /// End of file token + /// + Eof, + + /// + /// Whitespace token + /// + Whitespace, + + /// + /// Newline token + /// + Newline, + + /// + /// Any text not a whitespace or newline + /// + Raw, + + /// + /// Start of a code block "{{" + /// + CodeEnter, + + /// + /// End of a code block "}}" + /// + CodeExit, +} + +[StructLayout(LayoutKind.Auto)] +internal readonly record struct CharPosition(int Line, int Column, int Offset) +{ + public static readonly CharPosition Empty = new(0, 0, -1); +} + +[StructLayout(LayoutKind.Auto)] +internal readonly record struct Token(CharPosition Start, CharPosition End, TokenType Type) +{ + public static readonly Token Eof = new(CharPosition.Empty, CharPosition.Empty, TokenType.Eof); + + public bool IsRawToken() + { + return Type is TokenType.Raw or TokenType.Whitespace or TokenType.Newline; + } + + public override string ToString() + { + return $"{Type}({Start}:{End})"; + } + + public ReadOnlySpan ToSpan(in ReadOnlySpan template, Token? end = null) + { + if (Type == TokenType.Eof) + { + return []; + } + + var endToken = end ?? this; + return template.Slice( + Start.Offset, + length: Math.Max(endToken.End.Offset - Start.Offset + 1, 0) + ); + } +} diff --git a/Cutout/Parser/Identifiers.cs b/Cutout/Parser/Identifiers.cs deleted file mode 100644 index 90333f8..0000000 --- a/Cutout/Parser/Identifiers.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace Cutout.Parser; - -internal static class Identifiers -{ - public static readonly char[] Var = ['v', 'a', 'r']; - public static readonly char[] Call = ['c', 'a', 'l', 'l']; - public static readonly char[] If = ['i', 'f']; - - public static readonly char[] Else = ['e', 'l', 's', 'e']; - - public static readonly char[] End = ['e', 'n', 'd']; - - public static readonly char[] For = ['f', 'o', 'r']; - public static readonly char[] Foreach = ['f', 'o', 'r', 'e', 'a', 'c', 'h']; - public static readonly char[] While = ['w', 'h', 'i', 'l', 'e']; - public static readonly char[] Break = ['b', 'r', 'e', 'a', 'k']; - public static readonly char[] Continue = ['c', 'o', 'n', 't', 'i', 'n', 'u', 'e']; - public static readonly char[] Return = ['r', 'e', 't', 'u', 'r', 'n']; -} diff --git a/Cutout/Parser/Parser.cs b/Cutout/Parser/Parser.cs new file mode 100644 index 0000000..f466e75 --- /dev/null +++ b/Cutout/Parser/Parser.cs @@ -0,0 +1,318 @@ +using Cutout.Exceptions; +using Cutout.Extensions; + +namespace Cutout; + +internal static class Parser +{ + private static readonly char[] Var = ['v', 'a', 'r']; + private static readonly char[] Call = ['c', 'a', 'l', 'l']; + private static readonly char[] If = ['i', 'f']; + private static readonly char[] Else = ['e', 'l', 's', 'e']; + private static readonly char[] End = ['e', 'n', 'd']; + private static readonly char[] For = ['f', 'o', 'r']; + private static readonly char[] Foreach = ['f', 'o', 'r', 'e', 'a', 'c', 'h']; + private static readonly char[] While = ['w', 'h', 'i', 'l', 'e']; + private static readonly char[] Break = ['b', 'r', 'e', 'a', 'k']; + private static readonly char[] Continue = ['c', 'o', 'n', 't', 'i', 'n', 'u', 'e']; + private static readonly char[] Return = ['r', 'e', 't', 'u', 'r', 'n']; + + internal static SyntaxList Parse(TokenList tokens, in ReadOnlySpan template) + { + var index = 0; + return ParseInternal(tokens, in template, ref index); + } + + private static SyntaxList ParseInternal( + TokenList tokens, + in ReadOnlySpan template, + ref int index, + params char[][] breakOnAny + ) + { + var syntaxList = new SyntaxList(); + var tokenCount = tokens.Count; + + while (index < tokenCount) + { + var token = tokens[index]; + switch (token.Type) + { + case TokenType.Eof: + if (breakOnAny.Length > 0) + { + throw new ParseException( + token, + token.ToSpan(in template).ToString(), + breakOnAny.Length == 1 + ? $"Unexpected end of file, expected a {{{{ {new string(breakOnAny[0])} }}}} block" + : $"Unexpected end of file, expected one of {{{{ {string.Join(", ", breakOnAny.Select(b => new string(b)))} }}}} blocks" + ); + } + + index++; + continue; + case TokenType.CodeEnter: + { + ParseCodeBlock( + tokens, + in template, + ref index, + out var start, + out var count, + out var identifierIndex, + out var isJustIdentifier + ); + + var identifier = tokens[identifierIndex].ToSpan(in template); + + if (isJustIdentifier && breakOnAny.Length > 0) + { + for (var i = 0; i < breakOnAny.Length; i++) + { + var breakOn = breakOnAny[i]; + if (identifier.SequenceEqual(breakOn)) + { + return syntaxList; + } + } + } + + if (isJustIdentifier && identifier.SequenceEqual(Break)) + { + syntaxList.Add(Syntax.BreakStatement.Instance); + } + else if (isJustIdentifier && identifier.SequenceEqual(Continue)) + { + syntaxList.Add(Syntax.ContinueStatement.Instance); + } + else if (isJustIdentifier && identifier.SequenceEqual(Return)) + { + syntaxList.Add(Syntax.ReturnStatement.Instance); + } + else if (identifier.SequenceEqual(Var)) + { + var syntax = ParseVarStatement(ref identifier); + syntaxList.Add(syntax); + } + else if (identifier.SequenceEqual(Call)) + { + var syntax = ParseCallStatement(ref identifier, in template); + syntaxList.Add(syntax); + } + else if (identifier.SequenceEqual(While)) + { + ParseConditionalStatement( + ref identifier, + in template, + ref index, + out var condition, + out var expressions + ); + var syntax = new Syntax.WhileStatement(condition, expressions); + syntaxList.Add(syntax); + } + else if (identifier.SequenceEqual(For)) + { + ParseConditionalStatement( + ref identifier, + in template, + ref index, + out var condition, + out var expressions + ); + var syntax = new Syntax.ForStatement(condition, expressions); + syntaxList.Add(syntax); + } + else if (identifier.SequenceEqual(Foreach)) + { + ParseConditionalStatement( + ref identifier, + in template, + ref index, + out var condition, + out var expressions + ); + var syntax = new Syntax.ForeachStatement(condition, expressions); + syntaxList.Add(syntax); + } + else + { + var codeTokens = tokens.GetRange(start, count); + syntaxList.Add(new Syntax.RenderableExpression(codeTokens)); + } + break; + + TokenList RemainingTokens() => + tokens.GetRange(identifierIndex + 1, count - identifierIndex); + + Syntax.VarStatement ParseVarStatement(ref ReadOnlySpan identifier) + { + if (isJustIdentifier) + { + throw new ParseException( + tokens[identifierIndex], + identifier.ToString(), + "Variable declaration requires an expression" + ); + } + + var assignmentTokens = RemainingTokens(); + var syntax = new Syntax.VarStatement(assignmentTokens); + return syntax; + } + + Syntax.CallStatement ParseCallStatement( + ref ReadOnlySpan identifier, + in ReadOnlySpan template + ) + { + if (isJustIdentifier) + { + throw new ParseException( + tokens[identifierIndex], + identifier.ToString(), + "Call statement requires parameters" + ); + } + + var callTokens = RemainingTokens(); + var text = callTokens.ToSpan(in template).Trim().ToString(); + var callParts = text.Split( + ['(', ')'], + StringSplitOptions.RemoveEmptyEntries + ); + + if (callParts.Length != 2 || string.IsNullOrWhiteSpace(callParts[0])) + { + throw new ParseException( + tokens[identifierIndex], + identifier.ToString(), + "Call statement requires a function name and () with optional parameters" + ); + } + + return new Syntax.CallStatement( + callParts[0], + callParts[1].Split(',').Select(p => p.Trim()).ToArray() + ); + } + + void ParseConditionalStatement( + ref ReadOnlySpan identifier, + in ReadOnlySpan template, + ref int index, + out TokenList condition, + out SyntaxList expressions + ) + { + if (isJustIdentifier) + { + throw new ParseException( + tokens[identifierIndex], + identifier.ToString(), + $"{identifier.ToString()} statement requires a condition" + ); + } + + condition = RemainingTokens(); + expressions = ParseInternal( + tokens, + in template, + ref index, + breakOnAny: End + ); + } + } + default: + { + var rawText = ParseRawText(tokens, ref index); + syntaxList.Add(rawText); + break; + } + } + } + return syntaxList; + } + + private static void ParseCodeBlock( + TokenList tokens, + in ReadOnlySpan template, + ref int index, + out int start, + out int count, + out int identifierIndex, + out bool isJustIdentifier + ) + { + start = ++index; + var tokenCount = tokens.Count; + var rawTextCount = 0; + identifierIndex = -1; + var codeExitIndex = -1; + + while (index < tokenCount) + { + var token = tokens[index]; + rawTextCount += token.Type == TokenType.Raw ? 1 : 0; + + if (token.Type == TokenType.CodeExit) + { + codeExitIndex = index; + break; + } + + if (token.Type == TokenType.CodeEnter) + { + throw new ParseException( + token, + token.ToSpan(in template).ToString(), + "Nested code blocks are not allowed" + ); + } + + if (identifierIndex < 0 && token.Type == TokenType.Raw) + { + identifierIndex = index; + } + + index++; + } + + if (identifierIndex < 0) + { + throw new ParseException( + tokens[start], + tokens[start].ToSpan(in template).ToString(), + "Code block is empty" + ); + } + + if (codeExitIndex < 0) + { + throw new ParseException( + tokens[tokenCount - 1], + tokens[tokenCount - 1].ToSpan(in template).ToString(), + "Code exit token not found" + ); + } + + count = index - start; + index++; // Move past the CodeExit token + isJustIdentifier = rawTextCount == 1; + } + + private static Syntax.RawText ParseRawText(TokenList tokens, ref int index) + { + var start = index; + while (index < tokens.Count) + { + var type = tokens[index]; + if (!type.IsRawToken()) + break; + index++; + } + var rawTokens = tokens.GetRange(start, index - start); + return new Syntax.RawText(rawTokens); + } +} diff --git a/Cutout/Parser/Syntax.cs b/Cutout/Parser/Syntax.cs index 50c02fb..1191b85 100644 --- a/Cutout/Parser/Syntax.cs +++ b/Cutout/Parser/Syntax.cs @@ -1,139 +1,68 @@ -namespace Cutout.Parser; +namespace Cutout; internal abstract record Syntax { private Syntax() { } - public virtual bool SuppressTrailingNewline => true; + internal sealed record RawText(TokenList Value) : Syntax; - /// - /// Represents a raw text node in the template - /// - public sealed record RawText(string Value) : Syntax - { - public override bool SuppressTrailingNewline => false; - public bool ContainsNewline => Value.IndexOf('\n') != -1; - } - - /// - /// Represents a renderable expression in the template - /// - public sealed record RenderableExpression(string Value) : Syntax - { - public override bool SuppressTrailingNewline => false; - } + internal sealed record RenderableExpression(TokenList Value) : Syntax; - public abstract record WrappingExpressionsStatement(IReadOnlyList Expressions) : Syntax; + internal abstract record WrappingExpressionsStatement(IReadOnlyList Expressions) + : Syntax; - public abstract record ConditionalStatement(string Condition, IReadOnlyList Expressions) - : WrappingExpressionsStatement(Expressions); + internal abstract record ConditionalStatement( + TokenList Condition, + IReadOnlyList Expressions + ) : WrappingExpressionsStatement(Expressions); - /// - /// Represents a conditional if statement in the template - /// - /// condition to evaluate - /// expressions to render if the condition is true - public sealed record IfStatement(string Condition, IReadOnlyList Expressions) + internal sealed record IfStatement(TokenList Condition, IReadOnlyList Expressions) : ConditionalStatement(Condition, Expressions); - /// - /// Represents a conditional if else statement in the template - /// - /// condition to evaluate - /// expressions to render if the condition is true - public sealed record ElseIfStatement(string Condition, IReadOnlyList Expressions) + internal sealed record ElseIfStatement(TokenList Condition, IReadOnlyList Expressions) : ConditionalStatement(Condition, Expressions); - /// - /// Represents a conditional else statement in the template - /// - /// - public sealed record ElseStatement(IReadOnlyList Expressions) + internal sealed record ElseStatement(IReadOnlyList Expressions) : WrappingExpressionsStatement(Expressions); - /// - /// Represents a for statement in the template - /// - /// condition to evaluate; must be a valid for statement in C# - /// expressions to render if the condition is true - public sealed record ForStatement(string Condition, IReadOnlyList Expressions) + internal sealed record ForStatement(TokenList Condition, IReadOnlyList Expressions) : ConditionalStatement(Condition, Expressions); - /// - /// Represents a foreach statement in the template - /// - /// condition to evaluate; must be a valid foreach statement in C# - /// expressions to render if the condition is true - public sealed record ForeachStatement(string Condition, IReadOnlyList Expressions) + internal sealed record ForeachStatement(TokenList Condition, IReadOnlyList Expressions) : ConditionalStatement(Condition, Expressions); - /// - /// Represents a while statement in the template - /// - /// condition to evaluate; must be a valid while statement in C# - /// expressions to render if the condition is true - public sealed record WhileStatement(string Condition, IReadOnlyList Expressions) + internal sealed record WhileStatement(TokenList Condition, IReadOnlyList Expressions) : ConditionalStatement(Condition, Expressions); - /// - /// Represents a var statement in the template - /// - /// var statement to assign - public sealed record VarStatement(string Assignment) : Syntax - { - public override bool SuppressTrailingNewline => true; - } + internal sealed record VarStatement(TokenList Assignment) : Syntax; - /// - /// Represents a call to another template method - /// - /// name of the template method to call - /// parameters to pass to the template method - /// leading whitespace to add before the calls to raw or renderable expressions - public sealed record CallStatement(string Name, string Parameters, string LeadingWhitespace) - : Syntax - { - public override bool SuppressTrailingNewline => true; - public bool HasLeadingWhitespace => !string.IsNullOrEmpty(LeadingWhitespace); - } + internal sealed record CallStatement(string Name, IReadOnlyList Parameters) : Syntax; - /// - /// Represents a break statement in the template - /// - public sealed record BreakStatement : Syntax + internal sealed record BreakStatement : Syntax { private BreakStatement() { } - public static BreakStatement Instance { get; } = new(); + internal static BreakStatement Instance { get; } = new(); } - /// - /// Represents a continue statement in the template - /// - public sealed record ContinueStatement : Syntax + internal sealed record ContinueStatement : Syntax { private ContinueStatement() { } - public static ContinueStatement Instance { get; } = new(); + internal static ContinueStatement Instance { get; } = new(); } - /// - /// Represents a return statement in the template - /// - public sealed record ReturnStatement : Syntax + internal sealed record ReturnStatement : Syntax { private ReturnStatement() { } - public static ReturnStatement Instance { get; } = new(); + internal static ReturnStatement Instance { get; } = new(); } - /// - /// Represents a non-operation in the template, such as end - /// - public sealed record NoOp : Syntax + internal sealed record NoOp : Syntax { private NoOp() { } - public static NoOp Instance { get; } = new(); + internal static NoOp Instance { get; } = new(); } } diff --git a/Cutout/Parser/TemplateParser.CallStatement.cs b/Cutout/Parser/TemplateParser.CallStatement.cs deleted file mode 100644 index c2a19a9..0000000 --- a/Cutout/Parser/TemplateParser.CallStatement.cs +++ /dev/null @@ -1,77 +0,0 @@ -using Cutout.Exceptions; -using Cutout.Extensions; -using Scriban.Parsing; - -namespace Cutout.Parser; - -internal static partial class TemplateParser -{ - private static Syntax.CallStatement ParseCallStatement( - ReadOnlySpan tokens, - ReadOnlySpan template, - ref int index - ) - { - TryExtractWhitespace(tokens, template, index, out var whitespace); - ExtractCodeTokens(tokens, template, ref index, out var start, out var end); - - var fullExpression = start.ToSpan(template, end: end).ToString(); - var parts = fullExpression.Split(['(', ')'], StringSplitOptions.RemoveEmptyEntries); - if (parts.Length != 2) - { - throw new ParseException( - tokens[index], - tokens[index].ToSpan(template).ToString(), - "Invalid call statement. Expected format: 'MethodName(...)'." - ); - } - - var methodName = parts[0].Trim(); - if (string.IsNullOrEmpty(methodName)) - { - throw new ParseException( - tokens[index], - tokens[index].ToSpan(template).ToString(), - "Invalid call statement. Expected format: 'MethodName(...)'." - ); - } - - var parameters = parts[1].Trim(); - - TrySkipWhitespace(tokens, template, ref index); - return new Syntax.CallStatement(methodName, parameters, whitespace); - } - - private static void TryExtractWhitespace( - ReadOnlySpan tokens, - ReadOnlySpan template, - in int index, - out string whitespace - ) - { - whitespace = string.Empty; - - var lastToken = tokens[Math.Max(index - 3, 0)]; - if (lastToken.Type != TokenType.Raw) - { - return; - } - - var lastTokenSpan = lastToken.ToSpan(template); - var lastNewLineIndex = lastTokenSpan.LastIndexOf('\n'); - if (lastNewLineIndex == -1) - { - return; - } - - var newLineIndex = lastNewLineIndex + 1; - var whitespaceSlice = lastTokenSpan.Slice(newLineIndex); - - if (whitespaceSlice.IsEmpty || !whitespaceSlice.IsWhiteSpace()) - { - return; - } - - whitespace = whitespaceSlice.ToString(); - } -} diff --git a/Cutout/Parser/TemplateParser.EndBreakPredicate.cs b/Cutout/Parser/TemplateParser.EndBreakPredicate.cs deleted file mode 100644 index 4730fde..0000000 --- a/Cutout/Parser/TemplateParser.EndBreakPredicate.cs +++ /dev/null @@ -1,36 +0,0 @@ -using Cutout.Extensions; -using Scriban.Parsing; - -namespace Cutout.Parser; - -internal static partial class TemplateParser -{ - private sealed class EndParseContext : IParseContext - { - private EndParseContext(bool shouldSkipLeadingNewline, bool shouldSkipTrailingNewline) - { - ShouldSkipLeadingNewline = shouldSkipLeadingNewline; - ShouldSkipTrailingNewline = shouldSkipTrailingNewline; - } - - public static EndParseContext Instance { get; } = - new(shouldSkipLeadingNewline: true, shouldSkipTrailingNewline: true); - - public static EndParseContext InstanceWithSkipLeadingNewline { get; } = - new(shouldSkipLeadingNewline: true, shouldSkipTrailingNewline: false); - - public bool ShouldBreak( - ReadOnlySpan tokens, - ReadOnlySpan template, - ref int index - ) - { - var current = tokens[index]; - return current.ToSpan(template).SequenceEqual(Identifiers.End); - } - - public string MessageOnNoBreak => "end not found"; - public bool ShouldSkipLeadingNewline { get; } - public bool ShouldSkipTrailingNewline { get; } - } -} diff --git a/Cutout/Parser/TemplateParser.ForStatements.cs b/Cutout/Parser/TemplateParser.ForStatements.cs deleted file mode 100644 index adbb44e..0000000 --- a/Cutout/Parser/TemplateParser.ForStatements.cs +++ /dev/null @@ -1,60 +0,0 @@ -using Scriban.Parsing; - -namespace Cutout.Parser; - -internal static partial class TemplateParser -{ - private static Syntax.ForStatement ParseForStatement( - ReadOnlySpan tokens, - ReadOnlySpan template, - ref int index - ) - { - ExtractConditionalStatement( - "for", - tokens, - template, - EndParseContext.InstanceWithSkipLeadingNewline, - ref index, - out var condition, - out var expressions - ); - return new Syntax.ForStatement(condition, expressions); - } - - private static Syntax.ForeachStatement ParseForeachStatement( - ReadOnlySpan tokens, - ReadOnlySpan template, - ref int index - ) - { - ExtractConditionalStatement( - "foreach", - tokens, - template, - EndParseContext.InstanceWithSkipLeadingNewline, - ref index, - out var condition, - out var expressions - ); - return new Syntax.ForeachStatement(condition, expressions); - } - - private static Syntax.WhileStatement ParseWhileStatement( - ReadOnlySpan tokens, - ReadOnlySpan template, - ref int index - ) - { - ExtractConditionalStatement( - "while", - tokens, - template, - EndParseContext.InstanceWithSkipLeadingNewline, - ref index, - out var condition, - out var expressions - ); - return new Syntax.WhileStatement(condition, expressions); - } -} diff --git a/Cutout/Parser/TemplateParser.IParseContext.cs b/Cutout/Parser/TemplateParser.IParseContext.cs deleted file mode 100644 index 71b651b..0000000 --- a/Cutout/Parser/TemplateParser.IParseContext.cs +++ /dev/null @@ -1,15 +0,0 @@ -using Scriban.Parsing; - -namespace Cutout.Parser; - -internal static partial class TemplateParser -{ - internal interface IParseContext - { - bool ShouldBreak(ReadOnlySpan tokens, ReadOnlySpan template, ref int index); - string MessageOnNoBreak { get; } - - bool ShouldSkipLeadingNewline { get; } - bool ShouldSkipTrailingNewline { get; } - } -} diff --git a/Cutout/Parser/TemplateParser.IfStatements.cs b/Cutout/Parser/TemplateParser.IfStatements.cs deleted file mode 100644 index fb65392..0000000 --- a/Cutout/Parser/TemplateParser.IfStatements.cs +++ /dev/null @@ -1,111 +0,0 @@ -using Cutout.Exceptions; -using Cutout.Extensions; -using Scriban.Parsing; - -namespace Cutout.Parser; - -internal static partial class TemplateParser -{ - private sealed class IfParseContext : IParseContext - { - private IfParseContext() { } - - public static IfParseContext Instance { get; } = new(); - - public bool ShouldBreak( - ReadOnlySpan tokens, - ReadOnlySpan template, - ref int index - ) - { - var current = tokens[index]; - - if (!current.ToSpan(template).SequenceEqual(Identifiers.Else)) - { - return current.ToSpan(template).SequenceEqual(Identifiers.End); - } - - index -= 2; // rewind so that we can parse the else statement - return true; - } - - public string MessageOnNoBreak => "else or end not found"; - public bool ShouldSkipLeadingNewline => true; - public bool ShouldSkipTrailingNewline => true; - } - - private static Syntax.IfStatement ParseIfStatement( - ReadOnlySpan tokens, - ReadOnlySpan template, - ref int index - ) - { - ExtractCodeTokens(tokens, template, ref index, out var start, out var end); - var condition = start.ToSpan(template, end: end).ToString(); - - if (string.IsNullOrWhiteSpace(condition)) - { - throw new ParseException( - start, - start.ToSpan(template).ToString(), - "if statement condition not found" - ); - } - - var expressions = ParseInternal( - tokens, - template, - context: IfParseContext.Instance, - ref index - ); - return new Syntax.IfStatement(condition, expressions.ToArray()); - } - - private static Syntax ParseElseStatement( - ReadOnlySpan tokens, - ReadOnlySpan template, - ref int index - ) - { - var current = tokens[index]; - - IReadOnlyList expressions; - if (current.Type == TokenType.CodeExit) - { - index++; - expressions = ParseInternal( - tokens, - template, - context: EndParseContext.Instance, - ref index - ); - return new Syntax.ElseStatement(expressions); - } - - if (!current.ToSpan(template).SequenceEqual(Identifiers.If)) - { - throw new ParseException( - current, - current.ToSpan(template).ToString(), - "else statement must be followed by if statement" - ); - } - - index++; - - ExtractCodeTokens(tokens, template, ref index, out var start, out var end); - var condition = start.ToSpan(template, end: end).ToString(); - - if (string.IsNullOrWhiteSpace(condition)) - { - throw new ParseException( - start, - start.ToSpan(template).ToString(), - "else if statement condition not found" - ); - } - - expressions = ParseInternal(tokens, template, context: IfParseContext.Instance, ref index); - return new Syntax.ElseIfStatement(condition, expressions.ToArray()); - } -} diff --git a/Cutout/Parser/TemplateParser.KeywordStatement.cs b/Cutout/Parser/TemplateParser.KeywordStatement.cs deleted file mode 100644 index b5a7504..0000000 --- a/Cutout/Parser/TemplateParser.KeywordStatement.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Cutout.Exceptions; -using Cutout.Extensions; -using Scriban.Parsing; - -namespace Cutout.Parser; - -internal static partial class TemplateParser -{ - private static void EnsureKeywordStatementOnly( - string keyword, - ReadOnlySpan tokens, - ReadOnlySpan template, - in int index - ) - { - if (tokens[index].Type != TokenType.CodeExit) - { - throw new ParseException( - tokens[index], - tokens[index].ToSpan(template).ToString(), - $"Expected only keyword '{keyword}'" - ); - } - } -} diff --git a/Cutout/Parser/TemplateParser.VarStatement.cs b/Cutout/Parser/TemplateParser.VarStatement.cs deleted file mode 100644 index 2d7df6c..0000000 --- a/Cutout/Parser/TemplateParser.VarStatement.cs +++ /dev/null @@ -1,18 +0,0 @@ -using Cutout.Extensions; -using Scriban.Parsing; - -namespace Cutout.Parser; - -internal static partial class TemplateParser -{ - private static Syntax.VarStatement ParseVarStatement( - ReadOnlySpan tokens, - ReadOnlySpan template, - ref int index - ) - { - ExtractCodeTokens(tokens, template, ref index, out var start, out var end); - TrySkipWhitespace(tokens, template, ref index); - return new Syntax.VarStatement(start.ToSpan(template, end: end).ToString()); - } -} diff --git a/Cutout/Parser/TemplateParser.cs b/Cutout/Parser/TemplateParser.cs deleted file mode 100644 index 4f12606..0000000 --- a/Cutout/Parser/TemplateParser.cs +++ /dev/null @@ -1,338 +0,0 @@ -using Cutout.Exceptions; -using Cutout.Extensions; -using Scriban.Parsing; - -namespace Cutout.Parser; - -/// -/// Parser for the template syntax -/// -internal static partial class TemplateParser -{ - /// - /// Given a list of tokens, parse them into a list of syntax nodes - /// - /// tokens to parse - /// parsed template - /// list of syntax nodes - internal static IReadOnlyList Parse( - ReadOnlySpan tokens, - ReadOnlySpan template - ) - { - var index = 0; - return ParseInternal(tokens, template, context: null, ref index); - } - - private static IReadOnlyList ParseInternal( - ReadOnlySpan tokens, - ReadOnlySpan template, - IParseContext? context, - ref int index - ) - { - var result = new List(); - for (; index < tokens.Length; index++) - { - var token = tokens[index]; - - switch (token.Type) - { - case TokenType.Raw: - { - var syntax = ParseRawText( - template, - token, - shouldSkipLeadingNewline: context?.ShouldSkipLeadingNewline is true - && result.Count == 0 - ); - if (syntax != Syntax.NoOp.Instance) - { - result.Add(syntax); - } - break; - } - case TokenType.CodeEnter: - { - index++; - - if (context?.ShouldBreak(tokens, template, ref index) is true) - { - if ( - !context.ShouldSkipTrailingNewline - || result.Count == 0 - || result[result.Count - 1] is not Syntax.RawText rawText - ) - { - return result; - } - - var rawTextSpan = rawText.Value.AsSpan(); - rawTextSpan = TryRemoveTrailingNewline(rawTextSpan); - - if (rawTextSpan.Length == 0) - { - result.RemoveAt(result.Count - 1); - } - else - { - result[result.Count - 1] = new Syntax.RawText(rawTextSpan.ToString()); - } - - return result; - } - - var codeSyntax = ParseCode(tokens, template, ref index); - if (codeSyntax != Syntax.NoOp.Instance) - { - result.Add(codeSyntax); - } - - break; - } - case TokenType.CodeExit: - { - if (result.Count > 0 && result[result.Count - 1].SuppressTrailingNewline) - { - TrySkipWhitespace(tokens, template, ref index); - } - break; - } - } - } - - if (context == null) - { - return result; - } - - index -= 2; - throw new ParseException( - tokens[index], - tokens[index].ToSpan(template).ToString(), - context.MessageOnNoBreak - ); - } - - private static Syntax ParseRawText( - ReadOnlySpan template, - Token token, - bool shouldSkipLeadingNewline - ) - { - var rawText = token.ToSpan(template); - - if (shouldSkipLeadingNewline) - { - rawText = TryRemoveLeadingNewline(rawText); - } - - if (rawText.Length == 0) - { - return Syntax.NoOp.Instance; - } - - var syntax = new Syntax.RawText(rawText.ToString()); - return syntax; - } - - private static ReadOnlySpan TryRemoveTrailingNewline(ReadOnlySpan rawTextSpan) - { - if (rawTextSpan.Length < 2) - { - return rawTextSpan; - } - - var last2 = rawTextSpan.Slice(rawTextSpan.Length - 2); - if (last2.SequenceEqual(['\r', '\n'])) - { - return rawTextSpan.Slice(0, rawTextSpan.Length - 2); - } - - if (rawTextSpan[rawTextSpan.Length - 1] == '\n') - { - return rawTextSpan.Slice(0, rawTextSpan.Length - 1); - } - - return rawTextSpan; - } - - private static ReadOnlySpan TryRemoveLeadingNewline(ReadOnlySpan rawText) - { - var trimmed = rawText.TrimStart(' '); - return trimmed.Length switch - { - > 0 when trimmed[0] == '\n' => trimmed.Slice(1), - > 1 when trimmed[0] == '\r' && trimmed[1] == '\n' => trimmed.Slice(2), - _ => rawText, - }; - } - - private static Syntax ParseCode( - ReadOnlySpan tokens, - ReadOnlySpan template, - ref int index - ) - { - var current = tokens[index]; - var span = current.ToSpan(template); - - if (span.SequenceEqual(Identifiers.Var)) - { - index++; - return ParseVarStatement(tokens, template, ref index); - } - - if (span.SequenceEqual(Identifiers.Call)) - { - index++; - return ParseCallStatement(tokens, template, ref index); - } - - if (span.SequenceEqual(Identifiers.Break)) - { - EnsureKeywordStatementOnly("break", tokens, template, index: index + 1); - TrySkipWhitespace(tokens, template, ref index); - return Syntax.BreakStatement.Instance; - } - - if (span.SequenceEqual(Identifiers.Continue)) - { - EnsureKeywordStatementOnly("continue", tokens, template, index: index + 1); - TrySkipWhitespace(tokens, template, ref index); - return Syntax.ContinueStatement.Instance; - } - - if (span.SequenceEqual(Identifiers.Return)) - { - EnsureKeywordStatementOnly("return", tokens, template, index: index + 1); - TrySkipWhitespace(tokens, template, ref index); - return Syntax.ReturnStatement.Instance; - } - - if (span.SequenceEqual(Identifiers.If)) - { - index++; - return ParseIfStatement(tokens, template, ref index); - } - - if (span.SequenceEqual(Identifiers.Else)) - { - index++; - return ParseElseStatement(tokens, template, ref index); - } - - if (span.SequenceEqual(Identifiers.Foreach)) - { - index++; - return ParseForeachStatement(tokens, template, ref index); - } - - if (span.SequenceEqual(Identifiers.For)) - { - index++; - return ParseForStatement(tokens, template, ref index); - } - - if (span.SequenceEqual(Identifiers.While)) - { - index++; - return ParseWhileStatement(tokens, template, ref index); - } - - if (span.SequenceEqual(Identifiers.End)) - { - index++; - return Syntax.NoOp.Instance; - } - - return ParseRenderableExpression(tokens, template, ref index); - } - - private static Syntax.RenderableExpression ParseRenderableExpression( - ReadOnlySpan tokens, - ReadOnlySpan template, - ref int index - ) - { - ExtractCodeTokens(tokens, template, ref index, out var start, out var end); - var renderableExpression = start.ToSpan(template, end: end).ToString(); - return new Syntax.RenderableExpression(renderableExpression); - } - - private static void ExtractCodeTokens( - ReadOnlySpan tokens, - ReadOnlySpan template, - ref int index, - out Token start, - out Token end - ) - { - if (tokens[index].Type == TokenType.CodeEnter) - { - index++; - } - - start = tokens[index]; - - while (tokens.Length > index && tokens[index].Type != TokenType.CodeExit) - { - index++; - } - - var current = tokens[index]; - if (current.Type != TokenType.CodeExit) - { - throw new ParseException( - current, - current.ToSpan(template).ToString(), - "Code exit token not found" - ); - } - - index--; - - end = tokens[index]; - } - - private static void TrySkipWhitespace( - ReadOnlySpan tokens, - ReadOnlySpan template, - ref int index - ) - { - if ( - tokens.Length > index + 1 - && tokens[index + 1].Type == TokenType.Raw - && tokens[index + 1].ToSpan(template).IsWhiteSpace() - ) - { - index++; - } - } - - private static void ExtractConditionalStatement( - string keyword, - ReadOnlySpan tokens, - ReadOnlySpan template, - IParseContext context, - ref int index, - out string condition, - out IReadOnlyList expressions - ) - { - ExtractCodeTokens(tokens, template, ref index, out var start, out var end); - condition = start.ToSpan(template, end: end).ToString(); - - if (string.IsNullOrWhiteSpace(condition)) - { - throw new ParseException( - start, - start.ToSpan(template).ToString(), - $"{keyword} statement condition not found" - ); - } - - expressions = ParseInternal(tokens, template, context, ref index); - } -} diff --git a/Cutout/Renderer/Renderer.cs b/Cutout/Renderer/Renderer.cs index 84adaae..976b08c 100644 --- a/Cutout/Renderer/Renderer.cs +++ b/Cutout/Renderer/Renderer.cs @@ -80,7 +80,7 @@ bool includeWhitespaceReceiver ) { writer.Write("builder.Append(@\""); - writer.Write(rawText.Value); + // writer.Write(rawText.Value); writer.WriteLine("\");"); if (includeWhitespaceReceiver && rawText.ContainsNewline) diff --git a/Cutout/TemplateAttributeParts.cs b/Cutout/TemplateAttributeParts.cs index df164bb..1da66d1 100644 --- a/Cutout/TemplateAttributeParts.cs +++ b/Cutout/TemplateAttributeParts.cs @@ -1,7 +1,5 @@ using Cutout.Extensions; -using Cutout.Parser; using Microsoft.CodeAnalysis; -using Scriban.Parsing; namespace Cutout; @@ -25,8 +23,8 @@ public TemplateAttributeParts(MethodDetails details, SemanticModel ctxSemanticMo Template = template.HasValue ? template.Value?.ToString() : string.Empty; - var lexer = new Lexer(Template!); - var tokens = lexer.ToArray(); - Syntaxes = TemplateParser.Parse(tokens, Template.AsSpan()); + var templateSpan = Template.AsSpan(); + var tokens = Lexer.Tokenize(templateSpan); + // Syntaxes = TemplateParser.Parse(tokens, templateSpan); } } diff --git a/Cutout/TemplateSourceGenerator.TemplateMethodImpl.cs b/Cutout/TemplateSourceGenerator.TemplateMethodImpl.cs index d784a54..f9cfc47 100644 --- a/Cutout/TemplateSourceGenerator.TemplateMethodImpl.cs +++ b/Cutout/TemplateSourceGenerator.TemplateMethodImpl.cs @@ -94,10 +94,10 @@ bool includeWhitespaceReceiver return; } - foreach (var syntax in model.AttributeDetails.Syntaxes) - { - writer.WriteSyntax(syntax, includeWhitespaceReceiver); - } + // foreach (var syntax in model.AttributeDetails.Syntaxes) + // { + // writer.WriteSyntax(syntax, includeWhitespaceReceiver); + // } } private static void WriteNamespaceParts(IndentedTextWriter writer, TemplateMethodDetails model) diff --git a/Parent.Directory.Packages.props b/Parent.Directory.Packages.props index ca23aba..3f1e3c1 100644 --- a/Parent.Directory.Packages.props +++ b/Parent.Directory.Packages.props @@ -19,7 +19,6 @@ - From 80fa8b9c7a7ef7b8b7137e65b8d86c930aaebc87 Mon Sep 17 00:00:00 2001 From: bmazzarol Date: Sat, 26 Jul 2025 16:13:13 +0800 Subject: [PATCH 02/10] feat: if statement parsing WIP correct nested parsing not working yet, need a stack to store depth and pop --- Cutout.Tests/ParserTests.cs | 378 +++++++++++++++++++++++++++++++++++- Cutout/Parser/Parser.cs | 206 +++++++++++++++++--- 2 files changed, 548 insertions(+), 36 deletions(-) diff --git a/Cutout.Tests/ParserTests.cs b/Cutout.Tests/ParserTests.cs index 50be486..a3ffb85 100644 --- a/Cutout.Tests/ParserTests.cs +++ b/Cutout.Tests/ParserTests.cs @@ -268,7 +268,7 @@ public void Case12() var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): Variable declaration requires an expression (value: 'var')", + "Parse error at 1:4 (Raw): {{ var }} declaration requires an assignment expression (value: 'var')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -305,7 +305,7 @@ public void Case14() var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): Call statement requires parameters (value: 'call')", + "Parse error at 1:4 (Raw): {{ call }} statement requires parameters (value: 'call')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -319,7 +319,7 @@ public void Case15() var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): Call statement requires a function name and () with optional parameters (value: 'call')", + "Parse error at 1:4 (Raw): {{ call }} statement requires a function name and () with optional parameters (value: 'call')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -333,7 +333,7 @@ public void Case16() var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): Call statement requires a function name and () with optional parameters (value: 'call')", + "Parse error at 1:4 (Raw): {{ call }} statement requires a function name and () with optional parameters (value: 'call')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -407,7 +407,7 @@ public void Case18() var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): while statement requires a condition (value: 'while')", + "Parse error at 1:4 (Raw): {{ while }} statement requires a condition (value: 'while')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -459,4 +459,372 @@ public void Case21() var rawText = Assert.IsType(expression); Assert.Equal(" some code ", rawText.Value.ToSpan(template)); } + + [Fact(DisplayName = "An if statement can be parsed")] + public void Case22() + { + const string template = "{{ if condition }} some code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + var item = Assert.Single(result); + var ifStatement = Assert.IsType(item); + Assert.Equal(" condition ", ifStatement.Condition.ToSpan(template)); + + var expression = Assert.Single(ifStatement.Expressions); + var rawText = Assert.IsType(expression); + Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + } + + [Fact(DisplayName = "An if statement without a condition throws an error")] + public void Case23() + { + const string template = "{{ if }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): {{ if }} statement requires a condition (value: 'if')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("if", exception.Value); + } + + [Fact(DisplayName = "An if statement without an end token throws an error")] + public void Case24() + { + const string template = "{{ if condition }} some code"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at end of file: Unexpected end of file, expected a {{ else }} {{ else if }} or {{ end }} block", + exception.Message + ); + Assert.Equal(TokenType.Eof, exception.Token.Type); + Assert.Equal(string.Empty, exception.Value); + } + + [Fact(DisplayName = "An else without an if statement throws an error")] + public void Case25() + { + const string template = "{{ else }} some code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): {{ else }} statement requires a preceding {{ if }} or {{ elseif }} statement (value: 'else')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("else", exception.Value); + } + + [Fact(DisplayName = "An else if without an if statement throws an error")] + public void Case26() + { + const string template = "{{ else if condition }} some code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): {{ else }} statement requires a preceding {{ if }} or {{ elseif }} statement (value: 'else')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("else", exception.Value); + } + + [Fact(DisplayName = "An else statement can be parsed")] + public void Case27() + { + const string template = "{{ if condition }} some code {{ else }} other code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + Assert.Equal(2, result.Count); + var ifStatement = Assert.IsType(result[0]); + Assert.Equal(" condition ", ifStatement.Condition.ToSpan(template)); + + var expression = Assert.Single(ifStatement.Expressions); + var rawText = Assert.IsType(expression); + Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + + var elseStatement = Assert.IsType(result[1]); + expression = Assert.Single(elseStatement.Expressions); + var elseRawText = Assert.IsType(expression); + Assert.Equal(" other code ", elseRawText.Value.ToSpan(template)); + } + + [Fact(DisplayName = "An else after an else statement throws an error")] + public void Case28() + { + const string template = + "{{ if condition }} some code {{ else }} other code {{ else }} more code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:55 (Raw): Unexpected {{ else }} block, expected a {{ end }} block (value: 'else')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("else", exception.Value); + } + + [Fact(DisplayName = "An else if can be parsed")] + public void Case29() + { + const string template = + "{{ if condition }} some code {{ elseif otherCondition }} other code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + Assert.Equal(2, result.Count); + var ifStatement = Assert.IsType(result[0]); + Assert.Equal(" condition ", ifStatement.Condition.ToSpan(template)); + + var expression = Assert.Single(ifStatement.Expressions); + var rawText = Assert.IsType(expression); + Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + + var elseIfStatement = Assert.IsType(result[1]); + Assert.Equal(" otherCondition ", elseIfStatement.Condition.ToSpan(template)); + + expression = Assert.Single(elseIfStatement.Expressions); + var elseRawText = Assert.IsType(expression); + Assert.Equal(" other code ", elseRawText.Value.ToSpan(template)); + } + + [Fact(DisplayName = "An else if after an else statement throws an error")] + public void Case30() + { + const string template = + "{{ if condition }} some code {{ else }} other code {{ elseif anotherCondition }} more code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:55 (Raw): Unexpected {{ elseif }} block, expected a {{ end }} block (value: 'elseif')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("elseif", exception.Value); + } + + [Fact(DisplayName = "An else after and else statement throws an error")] + public void Case31() + { + const string template = + "{{ if condition }} some code {{ else }} other code {{ else }} final code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:55 (Raw): Unexpected {{ else }} block, expected a {{ end }} block (value: 'else')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("else", exception.Value); + } + + [Fact(DisplayName = "A if/else if/else statement can be parsed")] + public void Case32() + { + const string template = + "{{ if condition }} some code {{ elseif otherCondition }} other code {{ else }} final code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + Assert.Equal(3, result.Count); + + var ifStatement = Assert.IsType(result[0]); + Assert.Equal(" condition ", ifStatement.Condition.ToSpan(template)); + var expression = Assert.Single(ifStatement.Expressions); + var rawText = Assert.IsType(expression); + Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + + var elseIfStatement = Assert.IsType(result[1]); + Assert.Equal(" otherCondition ", elseIfStatement.Condition.ToSpan(template)); + expression = Assert.Single(elseIfStatement.Expressions); + rawText = Assert.IsType(expression); + Assert.Equal(" other code ", rawText.Value.ToSpan(template)); + + var elseStatement = Assert.IsType(result[2]); + expression = Assert.Single(elseStatement.Expressions); + rawText = Assert.IsType(expression); + Assert.Equal(" final code ", rawText.Value.ToSpan(template)); + } + + [Fact(DisplayName = "A end statement without a block throws an error")] + public void Case33() + { + const string template = "{{ end }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): {{ end }} found but not expected (value: 'end')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("end", exception.Value); + } + + [Fact(DisplayName = "A end statement with extra tokens throws an error")] + public void Case34() + { + const string template = "{{ end extra }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): {{ end }} statement should only contain the identifier (value: 'end')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("end", exception.Value); + } + + [Fact(DisplayName = "A elseif without a preceding if statement throws an error")] + public void Case35() + { + const string template = "{{ elseif condition }} some code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): {{ elseif }} statement requires a preceding {{ if }} or {{ elseif }} statement (value: 'elseif')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("elseif", exception.Value); + } + + [Fact(DisplayName = "A else statement with a condition throws an error")] + public void Case36() + { + const string template = "{{ if test }} something {{ else condition }} some code {{ end }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:28 (Raw): {{ else }} statement should only contain the identifier (value: 'else')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("else", exception.Value); + } + + [Fact(DisplayName = "A if statement can be nested inside an if statement")] + public void Case37() + { + const string template = + "{{ if condition1 }} some code {{ if condition2 }} nested code {{ end }}{{ end }} some more code"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + Assert.Equal(2, result.Count); + var ifStatement = Assert.IsType(result[0]); + Assert.Equal(" condition1 ", ifStatement.Condition.ToSpan(template)); + + Assert.Equal(2, ifStatement.Expressions.Count); + Assert.Collection( + ifStatement.Expressions, + expression => + { + var rawText = Assert.IsType(expression); + Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + }, + expression => + { + var nestedIfStatement = Assert.IsType(expression); + Assert.Equal(" condition2 ", nestedIfStatement.Condition.ToSpan(template)); + Assert.Single(nestedIfStatement.Expressions); + var nestedRawText = Assert.IsType(nestedIfStatement.Expressions[0]); + Assert.Equal(" nested code ", nestedRawText.Value.ToSpan(template)); + } + ); + + var rawText = Assert.IsType(result[1]); + Assert.Equal(" some more code", rawText.Value.ToSpan(template)); + } + + [Fact(DisplayName = "A if else statement can be nested inside an if else statement")] + public void Case38() + { + const string template = """ + {{ if condition1 }} + nested + {{ if condition2 }} + test + {{ else }} + other + {{ end }} + code + {{ else }} + final + {{ if test }} + test + {{ else }} + other + {{ end }} + code + {{ end }} + """; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + Assert.Equal(2, result.Count); + var ifStatement = Assert.IsType(result[0]); + Assert.Equal(" condition1 ", ifStatement.Condition.ToSpan(template)); + + Assert.Equal(2, ifStatement.Expressions.Count); + Assert.Collection( + ifStatement.Expressions, + expression => + { + var rawText = Assert.IsType(expression); + Assert.Equal(" nested ", rawText.Value.ToSpan(template)); + }, + expression => + { + var nestedIfStatement = Assert.IsType(expression); + Assert.Equal(" test ", nestedIfStatement.Condition.ToSpan(template)); + Assert.Equal(2, nestedIfStatement.Expressions.Count); + Assert.Collection( + nestedIfStatement.Expressions, + nestedExpression => + { + var nestedRawText = Assert.IsType(nestedExpression); + Assert.Equal(" test ", nestedRawText.Value.ToSpan(template)); + }, + nestedExpression => + { + var elseRawText = Assert.IsType(nestedExpression); + Assert.Equal(" other ", elseRawText.Value.ToSpan(template)); + } + ); + } + ); + var elseStatement = Assert.IsType(result[1]); + Assert.Equal(2, elseStatement.Expressions.Count); + Assert.Collection( + elseStatement.Expressions, + expression => + { + var rawText = Assert.IsType(expression); + Assert.Equal(" final ", rawText.Value.ToSpan(template)); + }, + expression => + { + var nestedIfStatement = Assert.IsType(expression); + Assert.Equal(" test ", nestedIfStatement.Condition.ToSpan(template)); + Assert.Equal(2, nestedIfStatement.Expressions.Count); + Assert.Collection( + nestedIfStatement.Expressions, + nestedExpression => + { + var nestedRawText = Assert.IsType(nestedExpression); + Assert.Equal(" test ", nestedRawText.Value.ToSpan(template)); + }, + nestedExpression => + { + var elseRawText = Assert.IsType(nestedExpression); + Assert.Equal(" other ", elseRawText.Value.ToSpan(template)); + } + ); + } + ); + } } diff --git a/Cutout/Parser/Parser.cs b/Cutout/Parser/Parser.cs index f466e75..d7a6481 100644 --- a/Cutout/Parser/Parser.cs +++ b/Cutout/Parser/Parser.cs @@ -9,6 +9,7 @@ internal static class Parser private static readonly char[] Call = ['c', 'a', 'l', 'l']; private static readonly char[] If = ['i', 'f']; private static readonly char[] Else = ['e', 'l', 's', 'e']; + private static readonly char[] ElseIf = ['e', 'l', 's', 'e', 'i', 'f']; private static readonly char[] End = ['e', 'n', 'd']; private static readonly char[] For = ['f', 'o', 'r']; private static readonly char[] Foreach = ['f', 'o', 'r', 'e', 'a', 'c', 'h']; @@ -20,14 +21,21 @@ internal static class Parser internal static SyntaxList Parse(TokenList tokens, in ReadOnlySpan template) { var index = 0; - return ParseInternal(tokens, in template, ref index); + return ParseInternal(tokens, in template, ref index, BreakOn.Eof); + } + + private enum BreakOn + { + Eof, + End, + Else, } private static SyntaxList ParseInternal( TokenList tokens, in ReadOnlySpan template, ref int index, - params char[][] breakOnAny + in BreakOn breakOn ) { var syntaxList = new SyntaxList(); @@ -39,14 +47,17 @@ params char[][] breakOnAny switch (token.Type) { case TokenType.Eof: - if (breakOnAny.Length > 0) + if (breakOn != BreakOn.Eof) { throw new ParseException( token, token.ToSpan(in template).ToString(), - breakOnAny.Length == 1 - ? $"Unexpected end of file, expected a {{{{ {new string(breakOnAny[0])} }}}} block" - : $"Unexpected end of file, expected one of {{{{ {string.Join(", ", breakOnAny.Select(b => new string(b)))} }}}} blocks" + breakOn switch + { + BreakOn.Else => + "Unexpected end of file, expected a {{ else }} {{ else if }} or {{ end }} block", + _ => "Unexpected end of file, expected a {{ end }} block", + } ); } @@ -66,16 +77,27 @@ out var isJustIdentifier var identifier = tokens[identifierIndex].ToSpan(in template); - if (isJustIdentifier && breakOnAny.Length > 0) + if (identifier.SequenceEqual(End)) { - for (var i = 0; i < breakOnAny.Length; i++) + if (!isJustIdentifier) { - var breakOn = breakOnAny[i]; - if (identifier.SequenceEqual(breakOn)) - { - return syntaxList; - } + throw new ParseException( + tokens[identifierIndex], + identifier.ToString(), + "{{ end }} statement should only contain the identifier" + ); } + + if (breakOn == BreakOn.Eof) + { + throw new ParseException( + tokens[identifierIndex], + identifier.ToString(), + "{{ end }} found but not expected" + ); + } + + return syntaxList; } if (isJustIdentifier && identifier.SequenceEqual(Break)) @@ -92,12 +114,12 @@ out var isJustIdentifier } else if (identifier.SequenceEqual(Var)) { - var syntax = ParseVarStatement(ref identifier); + var syntax = ParseVarStatement(in index, ref identifier); syntaxList.Add(syntax); } else if (identifier.SequenceEqual(Call)) { - var syntax = ParseCallStatement(ref identifier, in template); + var syntax = ParseCallStatement(in index, ref identifier, in template); syntaxList.Add(syntax); } else if (identifier.SequenceEqual(While)) @@ -106,6 +128,7 @@ out var isJustIdentifier ref identifier, in template, ref index, + BreakOn.End, out var condition, out var expressions ); @@ -118,6 +141,7 @@ out var expressions ref identifier, in template, ref index, + BreakOn.End, out var condition, out var expressions ); @@ -130,12 +154,119 @@ out var expressions ref identifier, in template, ref index, + BreakOn.End, out var condition, out var expressions ); var syntax = new Syntax.ForeachStatement(condition, expressions); syntaxList.Add(syntax); } + else if (identifier.SequenceEqual(If)) + { + ParseConditionalStatement( + ref identifier, + in template, + ref index, + BreakOn.Else, + out var condition, + out var expressions + ); + var syntax = new Syntax.IfStatement(condition, expressions); + syntaxList.Add(syntax); + } + else if (identifier.SequenceEqual(ElseIf)) + { + var lastSyntax = + syntaxList.Count > 0 ? syntaxList[syntaxList.Count - 1] : null; + if ( + breakOn == BreakOn.Eof + && lastSyntax is not Syntax.IfStatement + && lastSyntax is not Syntax.ElseIfStatement + ) + { + throw new ParseException( + tokens[identifierIndex], + identifier.ToString(), + "{{ elseif }} statement requires a preceding {{ if }} or {{ elseif }} statement" + ); + } + + switch (breakOn) + { + case BreakOn.End: + throw new ParseException( + tokens[identifierIndex], + identifier.ToString(), + "Unexpected {{ elseif }} block, expected a {{ end }} block" + ); + case BreakOn.Else: + { + RewindToToken(ref index, TokenType.CodeEnter); + return syntaxList; + } + } + + ParseConditionalStatement( + ref identifier, + in template, + ref index, + BreakOn.Else, + out var condition, + out var expressions + ); + var syntax = new Syntax.ElseIfStatement(condition, expressions); + syntaxList.Add(syntax); + } + else if (identifier.SequenceEqual(Else)) + { + var lastSyntax = + syntaxList.Count > 0 ? syntaxList[syntaxList.Count - 1] : null; + if ( + breakOn == BreakOn.Eof + && lastSyntax is not Syntax.IfStatement + && lastSyntax is not Syntax.ElseIfStatement + ) + { + throw new ParseException( + tokens[identifierIndex], + identifier.ToString(), + "{{ else }} statement requires a preceding {{ if }} or {{ elseif }} statement" + ); + } + + switch (breakOn) + { + case BreakOn.End: + throw new ParseException( + tokens[identifierIndex], + identifier.ToString(), + "Unexpected {{ else }} block, expected a {{ end }} block" + ); + case BreakOn.Else: + { + RewindToToken(ref index, TokenType.CodeEnter); + return syntaxList; + } + } + + if (!isJustIdentifier) + { + throw new ParseException( + tokens[identifierIndex], + identifier.ToString(), + "{{ else }} statement should only contain the identifier" + ); + } + + var expressions = ParseInternal( + tokens, + in template, + ref index, + BreakOn.End + ); + var syntax = new Syntax.ElseStatement(expressions); + syntaxList.Add(syntax); + } else { var codeTokens = tokens.GetRange(start, count); @@ -143,26 +274,35 @@ out var expressions } break; - TokenList RemainingTokens() => - tokens.GetRange(identifierIndex + 1, count - identifierIndex); + TokenList RemainingTokens(in int index) + { + var remainingCount = index - 2 - identifierIndex; + return remainingCount > 0 + ? tokens.GetRange(identifierIndex + 1, remainingCount) + : []; + } - Syntax.VarStatement ParseVarStatement(ref ReadOnlySpan identifier) + Syntax.VarStatement ParseVarStatement( + in int index, + ref ReadOnlySpan identifier + ) { if (isJustIdentifier) { throw new ParseException( tokens[identifierIndex], identifier.ToString(), - "Variable declaration requires an expression" + "{{ var }} declaration requires an assignment expression" ); } - var assignmentTokens = RemainingTokens(); + var assignmentTokens = RemainingTokens(in index); var syntax = new Syntax.VarStatement(assignmentTokens); return syntax; } Syntax.CallStatement ParseCallStatement( + in int index, ref ReadOnlySpan identifier, in ReadOnlySpan template ) @@ -172,11 +312,11 @@ in ReadOnlySpan template throw new ParseException( tokens[identifierIndex], identifier.ToString(), - "Call statement requires parameters" + "{{ call }} statement requires parameters" ); } - var callTokens = RemainingTokens(); + var callTokens = RemainingTokens(in index); var text = callTokens.ToSpan(in template).Trim().ToString(); var callParts = text.Split( ['(', ')'], @@ -188,7 +328,7 @@ in ReadOnlySpan template throw new ParseException( tokens[identifierIndex], identifier.ToString(), - "Call statement requires a function name and () with optional parameters" + "{{ call }} statement requires a function name and () with optional parameters" ); } @@ -202,6 +342,7 @@ void ParseConditionalStatement( ref ReadOnlySpan identifier, in ReadOnlySpan template, ref int index, + in BreakOn breakOn, out TokenList condition, out SyntaxList expressions ) @@ -211,17 +352,12 @@ out SyntaxList expressions throw new ParseException( tokens[identifierIndex], identifier.ToString(), - $"{identifier.ToString()} statement requires a condition" + $"{{{{ {identifier.ToString()} }}}} statement requires a condition" ); } - condition = RemainingTokens(); - expressions = ParseInternal( - tokens, - in template, - ref index, - breakOnAny: End - ); + condition = RemainingTokens(in index); + expressions = ParseInternal(tokens, in template, ref index, breakOn); } } default: @@ -233,6 +369,14 @@ out SyntaxList expressions } } return syntaxList; + + void RewindToToken(ref int index, in TokenType type) + { + while (index > -1 && tokens[index].Type != type) + { + index--; + } + } } private static void ParseCodeBlock( From 3aa10b11ed952f3410a000d50a8ad2544144c0b9 Mon Sep 17 00:00:00 2001 From: bmazzarol Date: Sun, 27 Jul 2025 23:20:49 +0800 Subject: [PATCH 03/10] chore: refactor parser * Correctly parses if statements * Is easier to parse metally --- Cutout.Tests/LexerTests.cs | 106 ++-- Cutout.Tests/ParserTests.cs | 260 +++----- Cutout/Exceptions/ParseException.cs | 4 +- .../IndentedTextWriterExtensions.cs | 2 +- Cutout/Extensions/SyntaxExtensions.cs | 2 +- Cutout/Extensions/TokenListExtensions.cs | 6 +- Cutout/Lexer/Token.cs | 15 +- Cutout/Parser/Parser.Context.cs | 60 ++ Cutout/Parser/Parser.ParseCodeBlock.cs | 87 +++ Cutout/Parser/Parser.cs | 565 +++++++----------- Cutout/Parser/Syntax.cs | 8 +- Cutout/TemplateAttributeParts.cs | 3 +- ...plateSourceGenerator.TemplateMethodImpl.cs | 1 - Cutout/TemplateSourceGenerator.cs | 1 - 14 files changed, 545 insertions(+), 575 deletions(-) create mode 100644 Cutout/Parser/Parser.Context.cs create mode 100644 Cutout/Parser/Parser.ParseCodeBlock.cs diff --git a/Cutout.Tests/LexerTests.cs b/Cutout.Tests/LexerTests.cs index cca10a3..573912e 100644 --- a/Cutout.Tests/LexerTests.cs +++ b/Cutout.Tests/LexerTests.cs @@ -6,7 +6,7 @@ public class LexerTests public void Case1() { const string text = " \t\n\r "; - var tokens = Lexer.Tokenize(text.AsSpan()); + var tokens = Lexer.Tokenize(text); Assert.Collection( tokens, @@ -15,28 +15,28 @@ public void Case1() Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(new CharPosition(1, 1, 0), token.Start); Assert.Equal(new CharPosition(1, 3, 2), token.End); - Assert.Equal(" \t", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" \t", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Newline, token.Type); Assert.Equal(new CharPosition(1, 4, 3), token.Start); Assert.Equal(new CharPosition(1, 4, 3), token.End); - Assert.Equal("\n", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("\n", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(new CharPosition(2, 1, 4), token.Start); Assert.Equal(new CharPosition(2, 3, 6), token.End); - Assert.Equal("\r ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("\r ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Eof, token.Type); Assert.Equal(new CharPosition(0, 0, -1), token.Start); Assert.Equal(new CharPosition(0, 0, -1), token.End); - Assert.Equal(string.Empty, token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(string.Empty, token.ToSpan(text).ToString()); } ); } @@ -45,7 +45,7 @@ public void Case1() public void Case2() { const string text = "{{ code block }}"; - var tokens = Lexer.Tokenize(text.AsSpan()); + var tokens = Lexer.Tokenize(text); Assert.Collection( tokens, token => @@ -53,21 +53,21 @@ public void Case2() Assert.Equal(TokenType.CodeEnter, token.Type); Assert.Equal(0, token.Start.Offset); Assert.Equal(1, token.End.Offset); - Assert.Equal("{{", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("{{", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(2, token.Start.Offset); Assert.Equal(2, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(3, token.Start.Offset); Assert.Equal(6, token.End.Offset); - Assert.Equal("code", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("code", token.ToSpan(text).ToString()); }, token => { @@ -75,28 +75,28 @@ public void Case2() Assert.Equal(7, token.Start.Offset); Assert.Equal(7, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(8, token.Start.Offset); Assert.Equal(12, token.End.Offset); - Assert.Equal("block", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("block", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(13, token.Start.Offset); Assert.Equal(13, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.CodeExit, token.Type); Assert.Equal(14, token.Start.Offset); Assert.Equal(15, token.End.Offset); - Assert.Equal("}}", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("}}", token.ToSpan(text).ToString()); }, token => { @@ -111,7 +111,7 @@ public void Case2() public void Case3() { const string text = "This is some raw text"; - var tokens = Lexer.Tokenize(text.AsSpan()); + var tokens = Lexer.Tokenize(text); Assert.Collection( tokens, token => @@ -119,63 +119,63 @@ public void Case3() Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(0, token.Start.Offset); Assert.Equal(3, token.End.Offset); - Assert.Equal("This", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("This", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(4, token.Start.Offset); Assert.Equal(4, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(5, token.Start.Offset); Assert.Equal(6, token.End.Offset); - Assert.Equal("is", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("is", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(7, token.Start.Offset); Assert.Equal(7, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(8, token.Start.Offset); Assert.Equal(11, token.End.Offset); - Assert.Equal("some", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("some", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(12, token.Start.Offset); Assert.Equal(12, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(13, token.Start.Offset); Assert.Equal(15, token.End.Offset); - Assert.Equal("raw", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("raw", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(16, token.Start.Offset); Assert.Equal(16, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(17, token.Start.Offset); Assert.Equal(20, token.End.Offset); - Assert.Equal("text", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("text", token.ToSpan(text).ToString()); }, token => { @@ -190,7 +190,7 @@ public void Case3() public void Case4() { const string text = "This is {{ code block }}\n some raw text\r\n and {{ more code }}"; - var tokens = Lexer.Tokenize(text.AsSpan()); + var tokens = Lexer.Tokenize(text); Assert.Collection( tokens, token => @@ -198,203 +198,203 @@ public void Case4() Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(0, token.Start.Offset); Assert.Equal(3, token.End.Offset); - Assert.Equal("This", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("This", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(4, token.Start.Offset); Assert.Equal(4, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(5, token.Start.Offset); Assert.Equal(6, token.End.Offset); - Assert.Equal("is", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("is", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(7, token.Start.Offset); Assert.Equal(7, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.CodeEnter, token.Type); Assert.Equal(8, token.Start.Offset); Assert.Equal(9, token.End.Offset); - Assert.Equal("{{", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("{{", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(10, token.Start.Offset); Assert.Equal(10, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(11, token.Start.Offset); Assert.Equal(14, token.End.Offset); - Assert.Equal("code", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("code", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(15, token.Start.Offset); Assert.Equal(15, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(16, token.Start.Offset); Assert.Equal(20, token.End.Offset); - Assert.Equal("block", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("block", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(21, token.Start.Offset); Assert.Equal(21, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.CodeExit, token.Type); Assert.Equal(22, token.Start.Offset); Assert.Equal(23, token.End.Offset); - Assert.Equal("}}", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("}}", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Newline, token.Type); Assert.Equal(24, token.Start.Offset); Assert.Equal(24, token.End.Offset); - Assert.Equal("\n", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("\n", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(25, token.Start.Offset); Assert.Equal(25, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(26, token.Start.Offset); Assert.Equal(29, token.End.Offset); - Assert.Equal("some", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("some", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(30, token.Start.Offset); Assert.Equal(30, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(31, token.Start.Offset); Assert.Equal(33, token.End.Offset); - Assert.Equal("raw", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("raw", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(34, token.Start.Offset); Assert.Equal(34, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(35, token.Start.Offset); Assert.Equal(38, token.End.Offset); - Assert.Equal("text", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("text", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Newline, token.Type); Assert.Equal(39, token.Start.Offset); Assert.Equal(40, token.End.Offset); - Assert.Equal("\r\n", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("\r\n", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(41, token.Start.Offset); Assert.Equal(41, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(42, token.Start.Offset); Assert.Equal(44, token.End.Offset); - Assert.Equal("and", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("and", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(45, token.Start.Offset); Assert.Equal(45, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.CodeEnter, token.Type); Assert.Equal(46, token.Start.Offset); Assert.Equal(47, token.End.Offset); - Assert.Equal("{{", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("{{", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(48, token.Start.Offset); Assert.Equal(48, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(49, token.Start.Offset); Assert.Equal(52, token.End.Offset); - Assert.Equal("more", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("more", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(53, token.Start.Offset); Assert.Equal(53, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(54, token.Start.Offset); Assert.Equal(57, token.End.Offset); - Assert.Equal("code", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("code", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); Assert.Equal(58, token.Start.Offset); Assert.Equal(58, token.End.Offset); - Assert.Equal(" ", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.CodeExit, token.Type); Assert.Equal(59, token.Start.Offset); Assert.Equal(60, token.End.Offset); - Assert.Equal("}}", token.ToSpan(text.AsSpan()).ToString()); + Assert.Equal("}}", token.ToSpan(text).ToString()); }, token => { diff --git a/Cutout.Tests/ParserTests.cs b/Cutout.Tests/ParserTests.cs index a3ffb85..78a49fd 100644 --- a/Cutout.Tests/ParserTests.cs +++ b/Cutout.Tests/ParserTests.cs @@ -1,5 +1,4 @@ using Cutout.Exceptions; -using Cutout.Extensions; namespace Cutout.Tests; @@ -75,58 +74,17 @@ public void Case3() item => { var rawText = Assert.IsType(item); - Assert.Collection( - rawText.Value, - token => - { - Assert.Equal(TokenType.Raw, token.Type); - Assert.Equal(token.ToSpan(template), "raw"); - }, - token => - { - Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); - } - ); + Assert.Equal("raw ", rawText.Value.ToSpan(template)); }, item => { var renderableExpression = Assert.IsType(item); - Assert.Collection( - renderableExpression.Value, - token => - { - Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); - }, - token => - { - Assert.Equal(TokenType.Raw, token.Type); - Assert.Equal(token.ToSpan(template), "code"); - }, - token => - { - Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); - } - ); + Assert.Equal(" code ", renderableExpression.Value.ToSpan(template)); }, item => { var rawText = Assert.IsType(item); - Assert.Collection( - rawText.Value, - token => - { - Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); - }, - token => - { - Assert.Equal(TokenType.Raw, token.Type); - Assert.Equal(token.ToSpan(template), "string"); - } - ); + Assert.Equal(" string", rawText.Value.ToSpan(template)); } ); } @@ -147,8 +105,11 @@ public void Case5() const string template = "{{ code"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); - Assert.Equal("Parse error at end of file: Code exit token not found", exception.Message); - Assert.Equal(string.Empty, exception.Token.ToSpan(template).ToString()); + Assert.Equal( + "Parse error at 1:4 (Raw): Code exit token not found (value: 'code')", + exception.Message + ); + Assert.Equal("code", exception.Token.ToSpan(template).ToString()); } [Fact(DisplayName = "Code blocks cannot be empty")] @@ -298,6 +259,19 @@ public void Case13() ); } + [Fact(DisplayName = "A call statement can be parsed (no arguments)")] + public void Case13b() + { + const string template = "{{ call function() }}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + + var item = Assert.Single(result); + var callStatement = Assert.IsType(item); + Assert.Equal("function", callStatement.Name); + Assert.Empty(callStatement.Parameters); + } + [Fact(DisplayName = "A call statement function part throws an error")] public void Case14() { @@ -326,6 +300,34 @@ public void Case15() Assert.Equal("call", exception.Value); } + [Fact(DisplayName = "A call statement without parentheses throws an error (first only)")] + public void Case15b() + { + const string template = "{{ call function( }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): {{ call }} statement requires a function name and () with optional parameters (value: 'call')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("call", exception.Value); + } + + [Fact(DisplayName = "A call statement without parentheses throws an error (last only)")] + public void Case15c() + { + const string template = "{{ call function) }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): {{ call }} statement requires a function name and () with optional parameters (value: 'call')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("call", exception.Value); + } + [Fact(DisplayName = "A call statement with only parentheses throws an error")] public void Case16() { @@ -497,7 +499,7 @@ public void Case24() var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at end of file: Unexpected end of file, expected a {{ else }} {{ else if }} or {{ end }} block", + "Parse error at end of file: Unexpected end of file, expected a {{ end }} block", exception.Message ); Assert.Equal(TokenType.Eof, exception.Token.Type); @@ -511,7 +513,7 @@ public void Case25() var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ else }} statement requires a preceding {{ if }} or {{ elseif }} statement (value: 'else')", + "Parse error at 1:4 (Raw): {{ else }} found but not expected (value: 'else')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -521,15 +523,15 @@ public void Case25() [Fact(DisplayName = "An else if without an if statement throws an error")] public void Case26() { - const string template = "{{ else if condition }} some code {{ end }}"; + const string template = "{{ elseif condition }} some code {{ end }}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ else }} statement requires a preceding {{ if }} or {{ elseif }} statement (value: 'else')", + "Parse error at 1:4 (Raw): {{ elseif }} found but not expected (value: 'elseif')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); - Assert.Equal("else", exception.Value); + Assert.Equal("elseif", exception.Value); } [Fact(DisplayName = "An else statement can be parsed")] @@ -539,18 +541,18 @@ public void Case27() var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); - Assert.Equal(2, result.Count); - var ifStatement = Assert.IsType(result[0]); + var single = Assert.Single(result); + var ifStatement = Assert.IsType(single); Assert.Equal(" condition ", ifStatement.Condition.ToSpan(template)); var expression = Assert.Single(ifStatement.Expressions); var rawText = Assert.IsType(expression); Assert.Equal(" some code ", rawText.Value.ToSpan(template)); - var elseStatement = Assert.IsType(result[1]); - expression = Assert.Single(elseStatement.Expressions); - var elseRawText = Assert.IsType(expression); - Assert.Equal(" other code ", elseRawText.Value.ToSpan(template)); + Assert.NotNull(ifStatement.Else); + expression = Assert.Single(ifStatement.Else.Expressions); + rawText = Assert.IsType(expression); + Assert.Equal(" other code ", rawText.Value.ToSpan(template)); } [Fact(DisplayName = "An else after an else statement throws an error")] @@ -561,7 +563,7 @@ public void Case28() var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:55 (Raw): Unexpected {{ else }} block, expected a {{ end }} block (value: 'else')", + "Parse error at 1:55 (Raw): Only one {{ else }} is allowed within an {{ if }} statement (value: 'else')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -576,20 +578,19 @@ public void Case29() var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); - Assert.Equal(2, result.Count); - var ifStatement = Assert.IsType(result[0]); + var single = Assert.Single(result); + var ifStatement = Assert.IsType(single); Assert.Equal(" condition ", ifStatement.Condition.ToSpan(template)); - var expression = Assert.Single(ifStatement.Expressions); var rawText = Assert.IsType(expression); Assert.Equal(" some code ", rawText.Value.ToSpan(template)); - - var elseIfStatement = Assert.IsType(result[1]); + Assert.NotNull(ifStatement.ElseIfs); + single = Assert.Single(ifStatement.ElseIfs); + var elseIfStatement = Assert.IsType(single); Assert.Equal(" otherCondition ", elseIfStatement.Condition.ToSpan(template)); - expression = Assert.Single(elseIfStatement.Expressions); - var elseRawText = Assert.IsType(expression); - Assert.Equal(" other code ", elseRawText.Value.ToSpan(template)); + rawText = Assert.IsType(expression); + Assert.Equal(" other code ", rawText.Value.ToSpan(template)); } [Fact(DisplayName = "An else if after an else statement throws an error")] @@ -600,28 +601,13 @@ public void Case30() var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:55 (Raw): Unexpected {{ elseif }} block, expected a {{ end }} block (value: 'elseif')", + "Parse error at 1:55 (Raw): Cannot have {{ elseif }} after {{ else }} in an {{ if }} statement (value: 'elseif')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); Assert.Equal("elseif", exception.Value); } - [Fact(DisplayName = "An else after and else statement throws an error")] - public void Case31() - { - const string template = - "{{ if condition }} some code {{ else }} other code {{ else }} final code {{ end }}"; - var tokens = Lexer.Tokenize(template); - var exception = Assert.Throws(() => Parser.Parse(tokens, template)); - Assert.Equal( - "Parse error at 1:55 (Raw): Unexpected {{ else }} block, expected a {{ end }} block (value: 'else')", - exception.Message - ); - Assert.Equal(TokenType.Raw, exception.Token.Type); - Assert.Equal("else", exception.Value); - } - [Fact(DisplayName = "A if/else if/else statement can be parsed")] public void Case32() { @@ -630,22 +616,21 @@ public void Case32() var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); - Assert.Equal(3, result.Count); - - var ifStatement = Assert.IsType(result[0]); + var single = Assert.Single(result); + var ifStatement = Assert.IsType(single); Assert.Equal(" condition ", ifStatement.Condition.ToSpan(template)); var expression = Assert.Single(ifStatement.Expressions); var rawText = Assert.IsType(expression); Assert.Equal(" some code ", rawText.Value.ToSpan(template)); - - var elseIfStatement = Assert.IsType(result[1]); + Assert.NotNull(ifStatement.ElseIfs); + single = Assert.Single(ifStatement.ElseIfs); + var elseIfStatement = Assert.IsType(single); Assert.Equal(" otherCondition ", elseIfStatement.Condition.ToSpan(template)); expression = Assert.Single(elseIfStatement.Expressions); rawText = Assert.IsType(expression); Assert.Equal(" other code ", rawText.Value.ToSpan(template)); - - var elseStatement = Assert.IsType(result[2]); - expression = Assert.Single(elseStatement.Expressions); + Assert.NotNull(ifStatement.Else); + expression = Assert.Single(ifStatement.Else.Expressions); rawText = Assert.IsType(expression); Assert.Equal(" final code ", rawText.Value.ToSpan(template)); } @@ -678,20 +663,6 @@ public void Case34() Assert.Equal("end", exception.Value); } - [Fact(DisplayName = "A elseif without a preceding if statement throws an error")] - public void Case35() - { - const string template = "{{ elseif condition }} some code {{ end }}"; - var tokens = Lexer.Tokenize(template); - var exception = Assert.Throws(() => Parser.Parse(tokens, template)); - Assert.Equal( - "Parse error at 1:4 (Raw): {{ elseif }} statement requires a preceding {{ if }} or {{ elseif }} statement (value: 'elseif')", - exception.Message - ); - Assert.Equal(TokenType.Raw, exception.Token.Type); - Assert.Equal("elseif", exception.Value); - } - [Fact(DisplayName = "A else statement with a condition throws an error")] public void Case36() { @@ -741,14 +712,14 @@ public void Case37() } [Fact(DisplayName = "A if else statement can be nested inside an if else statement")] - public void Case38() + public void Case39() { const string template = """ {{ if condition1 }} nested {{ if condition2 }} test - {{ else }} + {{ else }} other {{ end }} code @@ -765,66 +736,21 @@ public void Case38() var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); - Assert.Equal(2, result.Count); - var ifStatement = Assert.IsType(result[0]); + var single = Assert.Single(result); + var ifStatement = Assert.IsType(single); Assert.Equal(" condition1 ", ifStatement.Condition.ToSpan(template)); - - Assert.Equal(2, ifStatement.Expressions.Count); - Assert.Collection( - ifStatement.Expressions, - expression => - { - var rawText = Assert.IsType(expression); - Assert.Equal(" nested ", rawText.Value.ToSpan(template)); - }, - expression => - { - var nestedIfStatement = Assert.IsType(expression); - Assert.Equal(" test ", nestedIfStatement.Condition.ToSpan(template)); - Assert.Equal(2, nestedIfStatement.Expressions.Count); - Assert.Collection( - nestedIfStatement.Expressions, - nestedExpression => - { - var nestedRawText = Assert.IsType(nestedExpression); - Assert.Equal(" test ", nestedRawText.Value.ToSpan(template)); - }, - nestedExpression => - { - var elseRawText = Assert.IsType(nestedExpression); - Assert.Equal(" other ", elseRawText.Value.ToSpan(template)); - } - ); - } - ); - var elseStatement = Assert.IsType(result[1]); - Assert.Equal(2, elseStatement.Expressions.Count); - Assert.Collection( - elseStatement.Expressions, - expression => - { - var rawText = Assert.IsType(expression); - Assert.Equal(" final ", rawText.Value.ToSpan(template)); - }, - expression => - { - var nestedIfStatement = Assert.IsType(expression); - Assert.Equal(" test ", nestedIfStatement.Condition.ToSpan(template)); - Assert.Equal(2, nestedIfStatement.Expressions.Count); - Assert.Collection( - nestedIfStatement.Expressions, - nestedExpression => - { - var nestedRawText = Assert.IsType(nestedExpression); - Assert.Equal(" test ", nestedRawText.Value.ToSpan(template)); - }, - nestedExpression => - { - var elseRawText = Assert.IsType(nestedExpression); - Assert.Equal(" other ", elseRawText.Value.ToSpan(template)); - } - ); - } - ); + Assert.Equal(3, ifStatement.Expressions.Count); + var nestedIf = Assert.IsType(ifStatement.Expressions[1]); + Assert.Equal(" condition2 ", nestedIf.Condition.ToSpan(template)); + Assert.Single(nestedIf.Expressions); + Assert.NotNull(nestedIf.Else); + Assert.Single(nestedIf.Else.Expressions); + Assert.NotNull(ifStatement.Else); + Assert.Equal(3, ifStatement.Else.Expressions.Count); + var elseNestedIf = Assert.IsType(ifStatement.Else.Expressions[1]); + Assert.Equal(" test ", elseNestedIf.Condition.ToSpan(template)); + Assert.Single(elseNestedIf.Expressions); + Assert.NotNull(elseNestedIf.Else); + Assert.Single(elseNestedIf.Else.Expressions); } } diff --git a/Cutout/Exceptions/ParseException.cs b/Cutout/Exceptions/ParseException.cs index a59e3f8..8cfc359 100644 --- a/Cutout/Exceptions/ParseException.cs +++ b/Cutout/Exceptions/ParseException.cs @@ -1,6 +1,4 @@ -using System.Globalization; - -namespace Cutout.Exceptions; +namespace Cutout.Exceptions; internal sealed class ParseException : Exception { diff --git a/Cutout/Extensions/IndentedTextWriterExtensions.cs b/Cutout/Extensions/IndentedTextWriterExtensions.cs index d812d20..6906e5a 100644 --- a/Cutout/Extensions/IndentedTextWriterExtensions.cs +++ b/Cutout/Extensions/IndentedTextWriterExtensions.cs @@ -1,6 +1,6 @@ using System.CodeDom.Compiler; -namespace Cutout.Extensions; +namespace Cutout; internal static class IndentedTextWriterExtensions { diff --git a/Cutout/Extensions/SyntaxExtensions.cs b/Cutout/Extensions/SyntaxExtensions.cs index bbe9961..bd44724 100644 --- a/Cutout/Extensions/SyntaxExtensions.cs +++ b/Cutout/Extensions/SyntaxExtensions.cs @@ -2,7 +2,7 @@ using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; -namespace Cutout.Extensions; +namespace Cutout; internal static class SyntaxExtensions { diff --git a/Cutout/Extensions/TokenListExtensions.cs b/Cutout/Extensions/TokenListExtensions.cs index 1189c7f..f5ea819 100644 --- a/Cutout/Extensions/TokenListExtensions.cs +++ b/Cutout/Extensions/TokenListExtensions.cs @@ -1,8 +1,8 @@ -namespace Cutout.Extensions; +namespace Cutout; internal static class TokenListExtensions { - internal static ReadOnlySpan ToSpan(this TokenList list, in ReadOnlySpan template) + internal static ReadOnlySpan ToSpan(this TokenList list, string template) { if (list.Count == 0) { @@ -11,6 +11,6 @@ internal static ReadOnlySpan ToSpan(this TokenList list, in ReadOnlySpan ToSpan(in ReadOnlySpan template, Token? end = null) + public ReadOnlySpan ToSpan(string template, Token? end = null) { if (Type == TokenType.Eof) { @@ -64,9 +65,13 @@ public ReadOnlySpan ToSpan(in ReadOnlySpan template, Token? end = nu } var endToken = end ?? this; - return template.Slice( - Start.Offset, - length: Math.Max(endToken.End.Offset - Start.Offset + 1, 0) - ); + return template + .AsSpan() + .Slice(Start.Offset, length: Math.Max(endToken.End.Offset - Start.Offset + 1, 0)); + } + + internal ParseException Failure(string template, string reason) + { + return new ParseException(this, ToSpan(template).ToString(), reason); } } diff --git a/Cutout/Parser/Parser.Context.cs b/Cutout/Parser/Parser.Context.cs new file mode 100644 index 0000000..6ba3401 --- /dev/null +++ b/Cutout/Parser/Parser.Context.cs @@ -0,0 +1,60 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; +using Cutout.Exceptions; + +namespace Cutout; + +internal static partial class Parser +{ + private sealed class Context : IEnumerator + { + public TokenList Tokens { get; } + public string Template { get; } + public int Index { get; set; } + + public Context(TokenList tokens, string template) + { + Tokens = tokens; + Template = template; + Index = -1; + } + + public bool MoveNext() + { + if (Index + 1 >= Tokens.Count) + { + return false; + } + + Index++; + return true; + } + + [ExcludeFromCodeCoverage] + public void Reset() + { + Index = -1; + } + + [ExcludeFromCodeCoverage] + object IEnumerator.Current => Current; + + public Token Current => Tokens[Index]; + + public ParseException Failure(string reason) + { + return Current.Failure(Template, reason); + } + + public ParseException Failure(int index, string reason) + { + return Tokens[index].Failure(Template, reason); + } + + [ExcludeFromCodeCoverage] + public void Dispose() + { + Reset(); + } + } +} diff --git a/Cutout/Parser/Parser.ParseCodeBlock.cs b/Cutout/Parser/Parser.ParseCodeBlock.cs new file mode 100644 index 0000000..752e11c --- /dev/null +++ b/Cutout/Parser/Parser.ParseCodeBlock.cs @@ -0,0 +1,87 @@ +namespace Cutout; + +internal static partial class Parser +{ + private sealed class CodeBlockContext + { + private Context Context { get; set; } + public int StartIndex { get; private set; } + public int CodeExitIndex { get; set; } + public int Length { get; set; } + public int RawTextCount { get; set; } + public int IdentifierIndex { get; set; } + + public ReadOnlySpan Identifier => + Context.Tokens[IdentifierIndex].ToSpan(Context.Template); + public bool IsJustIdentifier => RawTextCount == 1; + + public CodeBlockContext(Context context) + { + Context = context; + Reset(context); + } + + public bool IsOnlyIdentifier(in ReadOnlySpan match) + { + return IsJustIdentifier && Identifier.SequenceEqual(match); + } + + public bool IsIdentifier(in ReadOnlySpan match) + { + return Identifier.SequenceEqual(match); + } + + public TokenList RemainingTokens() + { + var remainingCount = CodeExitIndex - 1 - IdentifierIndex; + return Context.Tokens.GetRange(IdentifierIndex + 1, remainingCount); + } + + public void Reset(Context context) + { + Context = context; + StartIndex = context.Index + 1; + CodeExitIndex = -1; + Length = 0; + IdentifierIndex = -1; + RawTextCount = 0; + } + } + + private static void ParseCodeBlock(Context context, CodeBlockContext codeBlockContext) + { + codeBlockContext.Reset(context); + while (context.MoveNext()) + { + codeBlockContext.RawTextCount += context.Current.Type == TokenType.Raw ? 1 : 0; + + if (context.Current.Type == TokenType.CodeExit) + { + codeBlockContext.CodeExitIndex = context.Index; + break; + } + + codeBlockContext.Length++; + + if (context.Current.Type == TokenType.CodeEnter) + { + throw context.Failure("Nested code blocks are not allowed"); + } + + if (codeBlockContext.IdentifierIndex < 0 && context.Current.Type == TokenType.Raw) + { + codeBlockContext.IdentifierIndex = context.Index; + } + } + + if (codeBlockContext.IdentifierIndex < 0) + { + throw context.Failure(codeBlockContext.StartIndex, "Code block is empty"); + } + + if (codeBlockContext.CodeExitIndex < 0) + { + throw context.Failure(codeBlockContext.Length - 1, "Code exit token not found"); + } + } +} diff --git a/Cutout/Parser/Parser.cs b/Cutout/Parser/Parser.cs index d7a6481..1ae4b41 100644 --- a/Cutout/Parser/Parser.cs +++ b/Cutout/Parser/Parser.cs @@ -1,9 +1,6 @@ -using Cutout.Exceptions; -using Cutout.Extensions; +namespace Cutout; -namespace Cutout; - -internal static class Parser +internal static partial class Parser { private static readonly char[] Var = ['v', 'a', 'r']; private static readonly char[] Call = ['c', 'a', 'l', 'l']; @@ -18,445 +15,341 @@ internal static class Parser private static readonly char[] Continue = ['c', 'o', 'n', 't', 'i', 'n', 'u', 'e']; private static readonly char[] Return = ['r', 'e', 't', 'u', 'r', 'n']; - internal static SyntaxList Parse(TokenList tokens, in ReadOnlySpan template) + internal static SyntaxList Parse(TokenList tokens, string template) { - var index = 0; - return ParseInternal(tokens, in template, ref index, BreakOn.Eof); + var context = new Context(tokens, template); + return ParseInternal(context, BreakOn.Eof, out _); } private enum BreakOn { Eof, End, - Else, } private static SyntaxList ParseInternal( - TokenList tokens, - in ReadOnlySpan template, - ref int index, - in BreakOn breakOn + Context context, + BreakOn breakOn, + out CodeBlockContext? endBlockContext ) { + CodeBlockContext? blockContext = null; var syntaxList = new SyntaxList(); - var tokenCount = tokens.Count; - while (index < tokenCount) + while (context.MoveNext()) { - var token = tokens[index]; - switch (token.Type) + switch (context.Current.Type) { case TokenType.Eof: if (breakOn != BreakOn.Eof) { - throw new ParseException( - token, - token.ToSpan(in template).ToString(), - breakOn switch - { - BreakOn.Else => - "Unexpected end of file, expected a {{ else }} {{ else if }} or {{ end }} block", - _ => "Unexpected end of file, expected a {{ end }} block", - } - ); + throw context.Failure("Unexpected end of file, expected a {{ end }} block"); } - index++; continue; case TokenType.CodeEnter: { - ParseCodeBlock( - tokens, - in template, - ref index, - out var start, - out var count, - out var identifierIndex, - out var isJustIdentifier - ); - - var identifier = tokens[identifierIndex].ToSpan(in template); + blockContext ??= new CodeBlockContext(context); + ParseCodeBlock(context, blockContext); - if (identifier.SequenceEqual(End)) + if (TryParseRecursiveEnd(context, blockContext, breakOn)) { - if (!isJustIdentifier) - { - throw new ParseException( - tokens[identifierIndex], - identifier.ToString(), - "{{ end }} statement should only contain the identifier" - ); - } - - if (breakOn == BreakOn.Eof) - { - throw new ParseException( - tokens[identifierIndex], - identifier.ToString(), - "{{ end }} found but not expected" - ); - } - + endBlockContext = blockContext; return syntaxList; } - if (isJustIdentifier && identifier.SequenceEqual(Break)) + if (blockContext.IsOnlyIdentifier(Break)) { syntaxList.Add(Syntax.BreakStatement.Instance); } - else if (isJustIdentifier && identifier.SequenceEqual(Continue)) + else if (blockContext.IsOnlyIdentifier(Continue)) { syntaxList.Add(Syntax.ContinueStatement.Instance); } - else if (isJustIdentifier && identifier.SequenceEqual(Return)) + else if (blockContext.IsOnlyIdentifier(Return)) { syntaxList.Add(Syntax.ReturnStatement.Instance); } - else if (identifier.SequenceEqual(Var)) + else if (TryParseVarStatement(context, blockContext, out var varSyntax)) { - var syntax = ParseVarStatement(in index, ref identifier); - syntaxList.Add(syntax); + syntaxList.Add(varSyntax); } - else if (identifier.SequenceEqual(Call)) + else if (TryParseCallStatement(context, blockContext, out var callSyntax)) { - var syntax = ParseCallStatement(in index, ref identifier, in template); - syntaxList.Add(syntax); + syntaxList.Add(callSyntax); } - else if (identifier.SequenceEqual(While)) + else if (blockContext.IsIdentifier(While)) { ParseConditionalStatement( - ref identifier, - in template, - ref index, - BreakOn.End, + context, + blockContext, out var condition, - out var expressions + out var expressions, + out _ ); var syntax = new Syntax.WhileStatement(condition, expressions); syntaxList.Add(syntax); } - else if (identifier.SequenceEqual(For)) + else if (blockContext.IsIdentifier(For)) { ParseConditionalStatement( - ref identifier, - in template, - ref index, - BreakOn.End, + context, + blockContext, out var condition, - out var expressions + out var expressions, + out _ ); var syntax = new Syntax.ForStatement(condition, expressions); syntaxList.Add(syntax); } - else if (identifier.SequenceEqual(Foreach)) + else if (blockContext.IsIdentifier(Foreach)) { ParseConditionalStatement( - ref identifier, - in template, - ref index, - BreakOn.End, + context, + blockContext, out var condition, - out var expressions + out var expressions, + out _ ); var syntax = new Syntax.ForeachStatement(condition, expressions); syntaxList.Add(syntax); } - else if (identifier.SequenceEqual(If)) + else if (TryParseIfStatement(context, blockContext, out var ifSyntax)) { - ParseConditionalStatement( - ref identifier, - in template, - ref index, - BreakOn.Else, - out var condition, - out var expressions - ); - var syntax = new Syntax.IfStatement(condition, expressions); - syntaxList.Add(syntax); + syntaxList.Add(ifSyntax); } - else if (identifier.SequenceEqual(ElseIf)) - { - var lastSyntax = - syntaxList.Count > 0 ? syntaxList[syntaxList.Count - 1] : null; - if ( - breakOn == BreakOn.Eof - && lastSyntax is not Syntax.IfStatement - && lastSyntax is not Syntax.ElseIfStatement - ) - { - throw new ParseException( - tokens[identifierIndex], - identifier.ToString(), - "{{ elseif }} statement requires a preceding {{ if }} or {{ elseif }} statement" - ); - } - - switch (breakOn) - { - case BreakOn.End: - throw new ParseException( - tokens[identifierIndex], - identifier.ToString(), - "Unexpected {{ elseif }} block, expected a {{ end }} block" - ); - case BreakOn.Else: - { - RewindToToken(ref index, TokenType.CodeEnter); - return syntaxList; - } - } - - ParseConditionalStatement( - ref identifier, - in template, - ref index, - BreakOn.Else, - out var condition, - out var expressions - ); - var syntax = new Syntax.ElseIfStatement(condition, expressions); - syntaxList.Add(syntax); - } - else if (identifier.SequenceEqual(Else)) + else { - var lastSyntax = - syntaxList.Count > 0 ? syntaxList[syntaxList.Count - 1] : null; - if ( - breakOn == BreakOn.Eof - && lastSyntax is not Syntax.IfStatement - && lastSyntax is not Syntax.ElseIfStatement - ) - { - throw new ParseException( - tokens[identifierIndex], - identifier.ToString(), - "{{ else }} statement requires a preceding {{ if }} or {{ elseif }} statement" - ); - } - - switch (breakOn) - { - case BreakOn.End: - throw new ParseException( - tokens[identifierIndex], - identifier.ToString(), - "Unexpected {{ else }} block, expected a {{ end }} block" - ); - case BreakOn.Else: - { - RewindToToken(ref index, TokenType.CodeEnter); - return syntaxList; - } - } - - if (!isJustIdentifier) - { - throw new ParseException( - tokens[identifierIndex], - identifier.ToString(), - "{{ else }} statement should only contain the identifier" - ); - } - - var expressions = ParseInternal( - tokens, - in template, - ref index, - BreakOn.End + var codeTokens = context.Tokens.GetRange( + blockContext.StartIndex, + blockContext.Length ); - var syntax = new Syntax.ElseStatement(expressions); + var syntax = new Syntax.RenderableExpression(codeTokens); syntaxList.Add(syntax); } - else - { - var codeTokens = tokens.GetRange(start, count); - syntaxList.Add(new Syntax.RenderableExpression(codeTokens)); - } break; - - TokenList RemainingTokens(in int index) - { - var remainingCount = index - 2 - identifierIndex; - return remainingCount > 0 - ? tokens.GetRange(identifierIndex + 1, remainingCount) - : []; - } - - Syntax.VarStatement ParseVarStatement( - in int index, - ref ReadOnlySpan identifier - ) - { - if (isJustIdentifier) - { - throw new ParseException( - tokens[identifierIndex], - identifier.ToString(), - "{{ var }} declaration requires an assignment expression" - ); - } - - var assignmentTokens = RemainingTokens(in index); - var syntax = new Syntax.VarStatement(assignmentTokens); - return syntax; - } - - Syntax.CallStatement ParseCallStatement( - in int index, - ref ReadOnlySpan identifier, - in ReadOnlySpan template - ) - { - if (isJustIdentifier) - { - throw new ParseException( - tokens[identifierIndex], - identifier.ToString(), - "{{ call }} statement requires parameters" - ); - } - - var callTokens = RemainingTokens(in index); - var text = callTokens.ToSpan(in template).Trim().ToString(); - var callParts = text.Split( - ['(', ')'], - StringSplitOptions.RemoveEmptyEntries - ); - - if (callParts.Length != 2 || string.IsNullOrWhiteSpace(callParts[0])) - { - throw new ParseException( - tokens[identifierIndex], - identifier.ToString(), - "{{ call }} statement requires a function name and () with optional parameters" - ); - } - - return new Syntax.CallStatement( - callParts[0], - callParts[1].Split(',').Select(p => p.Trim()).ToArray() - ); - } - - void ParseConditionalStatement( - ref ReadOnlySpan identifier, - in ReadOnlySpan template, - ref int index, - in BreakOn breakOn, - out TokenList condition, - out SyntaxList expressions - ) - { - if (isJustIdentifier) - { - throw new ParseException( - tokens[identifierIndex], - identifier.ToString(), - $"{{{{ {identifier.ToString()} }}}} statement requires a condition" - ); - } - - condition = RemainingTokens(in index); - expressions = ParseInternal(tokens, in template, ref index, breakOn); - } } default: { - var rawText = ParseRawText(tokens, ref index); + var rawText = ParseRawText(context); syntaxList.Add(rawText); break; } } } + + endBlockContext = null; return syntaxList; + } + + private static bool TryParseRecursiveEnd( + Context context, + CodeBlockContext codeBlockContext, + BreakOn breakOn + ) + { + if ( + !codeBlockContext.Identifier.SequenceEqual(End) + && !codeBlockContext.Identifier.SequenceEqual(ElseIf) + && !codeBlockContext.Identifier.SequenceEqual(Else) + ) + { + return false; + } - void RewindToToken(ref int index, in TokenType type) + if ( + !codeBlockContext.IsJustIdentifier && !codeBlockContext.Identifier.SequenceEqual(ElseIf) + ) { - while (index > -1 && tokens[index].Type != type) - { - index--; - } + throw context.Failure( + codeBlockContext.IdentifierIndex, + $"{{{{ {codeBlockContext.Identifier.ToString()} }}}} statement should only contain the identifier" + ); } + + if (breakOn == BreakOn.Eof) + { + throw context.Failure( + codeBlockContext.IdentifierIndex, + $"{{{{ {codeBlockContext.Identifier.ToString()} }}}} found but not expected" + ); + } + + return true; } - private static void ParseCodeBlock( - TokenList tokens, - in ReadOnlySpan template, - ref int index, - out int start, - out int count, - out int identifierIndex, - out bool isJustIdentifier + private static bool TryParseVarStatement( + Context context, + CodeBlockContext blockContext, + out Syntax.VarStatement? syntax ) { - start = ++index; - var tokenCount = tokens.Count; - var rawTextCount = 0; - identifierIndex = -1; - var codeExitIndex = -1; + if (!blockContext.Identifier.SequenceEqual(Var)) + { + syntax = null; + return false; + } - while (index < tokenCount) + if (blockContext.IsJustIdentifier) { - var token = tokens[index]; - rawTextCount += token.Type == TokenType.Raw ? 1 : 0; + throw context.Failure( + blockContext.IdentifierIndex, + "{{ var }} declaration requires an assignment expression" + ); + } - if (token.Type == TokenType.CodeExit) - { - codeExitIndex = index; - break; - } + syntax = new Syntax.VarStatement(blockContext.RemainingTokens()); + return true; + } - if (token.Type == TokenType.CodeEnter) - { - throw new ParseException( - token, - token.ToSpan(in template).ToString(), - "Nested code blocks are not allowed" - ); - } + private static bool TryParseCallStatement( + Context context, + CodeBlockContext blockContext, + out Syntax.CallStatement? syntax + ) + { + if (!blockContext.Identifier.SequenceEqual(Call)) + { + syntax = null; + return false; + } - if (identifierIndex < 0 && token.Type == TokenType.Raw) - { - identifierIndex = index; - } + if (blockContext.IsJustIdentifier) + { + throw context.Failure( + blockContext.IdentifierIndex, + "{{ call }} statement requires parameters" + ); + } + + var text = blockContext.RemainingTokens().ToSpan(context.Template).Trim().ToString(); + var callParts = text.Split('(', ')'); - index++; + if (callParts.Length != 3 || string.IsNullOrWhiteSpace(callParts[0])) + { + throw context.Failure( + blockContext.IdentifierIndex, + "{{ call }} statement requires a function name and () with optional parameters" + ); } - if (identifierIndex < 0) + syntax = new Syntax.CallStatement( + callParts[0], + callParts[1] + .Split(',') + .Select(p => p.Trim()) + .Where(p => !string.IsNullOrWhiteSpace(p)) + .ToArray() + ); + return true; + } + + private static void ParseConditionalStatement( + Context context, + CodeBlockContext blockContext, + out TokenList condition, + out SyntaxList expressions, + out CodeBlockContext? endBlockContext + ) + { + if (blockContext.IsJustIdentifier) { - throw new ParseException( - tokens[start], - tokens[start].ToSpan(in template).ToString(), - "Code block is empty" + throw context.Failure( + blockContext.IdentifierIndex, + $"{{{{ {blockContext.Identifier.ToString()} }}}} statement requires a condition" ); } - if (codeExitIndex < 0) + condition = blockContext.RemainingTokens(); + expressions = ParseInternal(context, BreakOn.End, out endBlockContext); + } + + private static bool TryParseIfStatement( + Context context, + CodeBlockContext blockContext, + out Syntax.IfStatement? syntax + ) + { + if (!blockContext.Identifier.SequenceEqual(If)) + { + syntax = null; + return false; + } + + if (blockContext.IsJustIdentifier) { - throw new ParseException( - tokens[tokenCount - 1], - tokens[tokenCount - 1].ToSpan(in template).ToString(), - "Code exit token not found" + throw context.Failure( + blockContext.IdentifierIndex, + "{{ if }} statement requires a condition" ); } - count = index - start; - index++; // Move past the CodeExit token - isJustIdentifier = rawTextCount == 1; + ParseConditionalStatement( + context, + blockContext, + out var condition, + out var expressions, + out var endBlockContext + ); + + List? elseifStatements = null; + Syntax.ElseStatement? elseStatement = null; + do + { + if (endBlockContext!.Identifier.SequenceEqual(ElseIf)) + { + if (elseStatement != null) + { + throw context.Failure( + endBlockContext.IdentifierIndex, + "Cannot have {{ elseif }} after {{ else }} in an {{ if }} statement" + ); + } + + ParseConditionalStatement( + context, + endBlockContext, + out var elseifCondition, + out var elseifExpressions, + out endBlockContext + ); + elseifStatements ??= []; + elseifStatements.Add( + new Syntax.ElseIfStatement(elseifCondition, elseifExpressions) + ); + } + else if (endBlockContext.Identifier.SequenceEqual(Else)) + { + if (elseStatement != null) + { + throw context.Failure( + endBlockContext.IdentifierIndex, + "Only one {{ else }} is allowed within an {{ if }} statement" + ); + } + + var elseExpressions = ParseInternal(context, BreakOn.End, out endBlockContext); + elseStatement = new Syntax.ElseStatement(elseExpressions); + } + } while (!endBlockContext!.Identifier.SequenceEqual(End)); + + syntax = new Syntax.IfStatement(condition, expressions, elseifStatements, elseStatement); + + return true; } - private static Syntax.RawText ParseRawText(TokenList tokens, ref int index) + private static Syntax.RawText ParseRawText(Context context) { - var start = index; - while (index < tokens.Count) + var start = context.Index; + while (context.MoveNext()) { - var type = tokens[index]; - if (!type.IsRawToken()) + if (!context.Current.IsRawToken()) + { break; - index++; + } } - var rawTokens = tokens.GetRange(start, index - start); + var rawTokens = context.Tokens.GetRange(start, context.Index - start); + context.Index--; return new Syntax.RawText(rawTokens); } } diff --git a/Cutout/Parser/Syntax.cs b/Cutout/Parser/Syntax.cs index 1191b85..ab58371 100644 --- a/Cutout/Parser/Syntax.cs +++ b/Cutout/Parser/Syntax.cs @@ -16,8 +16,12 @@ internal abstract record ConditionalStatement( IReadOnlyList Expressions ) : WrappingExpressionsStatement(Expressions); - internal sealed record IfStatement(TokenList Condition, IReadOnlyList Expressions) - : ConditionalStatement(Condition, Expressions); + internal sealed record IfStatement( + TokenList Condition, + IReadOnlyList Expressions, + IReadOnlyList? ElseIfs = null, + ElseStatement? Else = null + ) : ConditionalStatement(Condition, Expressions); internal sealed record ElseIfStatement(TokenList Condition, IReadOnlyList Expressions) : ConditionalStatement(Condition, Expressions); diff --git a/Cutout/TemplateAttributeParts.cs b/Cutout/TemplateAttributeParts.cs index 1da66d1..925cdbb 100644 --- a/Cutout/TemplateAttributeParts.cs +++ b/Cutout/TemplateAttributeParts.cs @@ -1,5 +1,4 @@ -using Cutout.Extensions; -using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis; namespace Cutout; diff --git a/Cutout/TemplateSourceGenerator.TemplateMethodImpl.cs b/Cutout/TemplateSourceGenerator.TemplateMethodImpl.cs index f9cfc47..e86e1eb 100644 --- a/Cutout/TemplateSourceGenerator.TemplateMethodImpl.cs +++ b/Cutout/TemplateSourceGenerator.TemplateMethodImpl.cs @@ -1,5 +1,4 @@ using System.CodeDom.Compiler; -using Cutout.Extensions; using Microsoft.CodeAnalysis; namespace Cutout; diff --git a/Cutout/TemplateSourceGenerator.cs b/Cutout/TemplateSourceGenerator.cs index 11a62a6..0516a58 100644 --- a/Cutout/TemplateSourceGenerator.cs +++ b/Cutout/TemplateSourceGenerator.cs @@ -1,6 +1,5 @@ using System.CodeDom.Compiler; using System.Text; -using Cutout.Extensions; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; From e4a1fdc240e178928026bcbf8462d0886babcbd5 Mon Sep 17 00:00:00 2001 From: bmazzarol Date: Sun, 27 Jul 2025 23:29:53 +0800 Subject: [PATCH 04/10] fix: remove Context allocations --- Cutout/Parser/Parser.Context.cs | 11 +++++++++-- Cutout/Parser/Parser.cs | 14 +++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/Cutout/Parser/Parser.Context.cs b/Cutout/Parser/Parser.Context.cs index 6ba3401..4ef9d87 100644 --- a/Cutout/Parser/Parser.Context.cs +++ b/Cutout/Parser/Parser.Context.cs @@ -8,8 +8,8 @@ internal static partial class Parser { private sealed class Context : IEnumerator { - public TokenList Tokens { get; } - public string Template { get; } + public TokenList Tokens { get; private set; } + public string Template { get; private set; } public int Index { get; set; } public Context(TokenList tokens, string template) @@ -56,5 +56,12 @@ public void Dispose() { Reset(); } + + public void Reset(TokenList tokens, string template) + { + Tokens = tokens; + Template = template; + Index = -1; + } } } diff --git a/Cutout/Parser/Parser.cs b/Cutout/Parser/Parser.cs index 1ae4b41..86e267b 100644 --- a/Cutout/Parser/Parser.cs +++ b/Cutout/Parser/Parser.cs @@ -15,9 +15,21 @@ internal static partial class Parser private static readonly char[] Continue = ['c', 'o', 'n', 't', 'i', 'n', 'u', 'e']; private static readonly char[] Return = ['r', 'e', 't', 'u', 'r', 'n']; + [ThreadStatic] + private static Context? _threadContext; + internal static SyntaxList Parse(TokenList tokens, string template) { - var context = new Context(tokens, template); + var context = _threadContext; + if (context == null) + { + context = new Context(tokens, template); + _threadContext = context; + } + else + { + context.Reset(tokens, template); + } return ParseInternal(context, BreakOn.Eof, out _); } From bfc8e784a19e6261e710a4759beaf20a86daef66 Mon Sep 17 00:00:00 2001 From: bmazzarol Date: Mon, 28 Jul 2025 17:46:03 +0800 Subject: [PATCH 05/10] feat: improve parser --- Cutout/Parser/Parser.cs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Cutout/Parser/Parser.cs b/Cutout/Parser/Parser.cs index 86e267b..6c4031a 100644 --- a/Cutout/Parser/Parser.cs +++ b/Cutout/Parser/Parser.cs @@ -161,17 +161,15 @@ BreakOn breakOn ) { if ( - !codeBlockContext.Identifier.SequenceEqual(End) - && !codeBlockContext.Identifier.SequenceEqual(ElseIf) - && !codeBlockContext.Identifier.SequenceEqual(Else) + !codeBlockContext.IsIdentifier(End) + && !codeBlockContext.IsIdentifier(ElseIf) + && !codeBlockContext.IsIdentifier(Else) ) { return false; } - if ( - !codeBlockContext.IsJustIdentifier && !codeBlockContext.Identifier.SequenceEqual(ElseIf) - ) + if (!codeBlockContext.IsOnlyIdentifier(ElseIf)) { throw context.Failure( codeBlockContext.IdentifierIndex, @@ -196,7 +194,7 @@ private static bool TryParseVarStatement( out Syntax.VarStatement? syntax ) { - if (!blockContext.Identifier.SequenceEqual(Var)) + if (!blockContext.IsIdentifier(Var)) { syntax = null; return false; @@ -220,7 +218,7 @@ private static bool TryParseCallStatement( out Syntax.CallStatement? syntax ) { - if (!blockContext.Identifier.SequenceEqual(Call)) + if (!blockContext.IsIdentifier(Call)) { syntax = null; return false; @@ -282,7 +280,7 @@ private static bool TryParseIfStatement( out Syntax.IfStatement? syntax ) { - if (!blockContext.Identifier.SequenceEqual(If)) + if (!blockContext.IsIdentifier(If)) { syntax = null; return false; @@ -308,7 +306,7 @@ out var endBlockContext Syntax.ElseStatement? elseStatement = null; do { - if (endBlockContext!.Identifier.SequenceEqual(ElseIf)) + if (endBlockContext!.IsIdentifier(ElseIf)) { if (elseStatement != null) { @@ -330,7 +328,7 @@ out endBlockContext new Syntax.ElseIfStatement(elseifCondition, elseifExpressions) ); } - else if (endBlockContext.Identifier.SequenceEqual(Else)) + else if (endBlockContext.IsIdentifier(Else)) { if (elseStatement != null) { @@ -343,7 +341,7 @@ out endBlockContext var elseExpressions = ParseInternal(context, BreakOn.End, out endBlockContext); elseStatement = new Syntax.ElseStatement(elseExpressions); } - } while (!endBlockContext!.Identifier.SequenceEqual(End)); + } while (!endBlockContext!.IsIdentifier(End)); syntax = new Syntax.IfStatement(condition, expressions, elseifStatements, elseStatement); From 58c0b80ba52d5514d96b1e9f9b48922ca1f63af2 Mon Sep 17 00:00:00 2001 From: bmazzarol Date: Mon, 28 Jul 2025 22:38:19 +0800 Subject: [PATCH 06/10] fix: failing tests --- Cutout/Parser/Parser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cutout/Parser/Parser.cs b/Cutout/Parser/Parser.cs index 6c4031a..4b9edae 100644 --- a/Cutout/Parser/Parser.cs +++ b/Cutout/Parser/Parser.cs @@ -169,7 +169,7 @@ BreakOn breakOn return false; } - if (!codeBlockContext.IsOnlyIdentifier(ElseIf)) + if (!codeBlockContext.IsJustIdentifier && !codeBlockContext.IsIdentifier(ElseIf)) { throw context.Failure( codeBlockContext.IdentifierIndex, From 366d8755023b0046ac0dc5c085e89dfc3093ab29 Mon Sep 17 00:00:00 2001 From: bmazzarol Date: Sat, 2 Aug 2025 13:23:20 +0800 Subject: [PATCH 07/10] fix: add support for standard code blocks and ws suppression --- Cutout.Tests/LexerTests.cs | 113 +++++-- Cutout.Tests/ParserTests.cs | 239 +++++++++------ Cutout/Extensions/TokenListExtensions.cs | 25 +- .../Lexer/Lexer.ApplyWhitespaceSuppression.cs | 64 ++++ Cutout/Lexer/Lexer.Context.cs | 90 ++++++ Cutout/Lexer/Lexer.cs | 275 ++++++++---------- Cutout/Lexer/Token.cs | 81 +++++- Cutout/Parser/Parser.ParseBlock.cs | 103 +++++++ Cutout/Parser/Parser.ParseCodeBlock.cs | 87 ------ Cutout/Parser/Parser.cs | 77 +++-- Cutout/Parser/Syntax.cs | 5 +- Cutout/README.md | 37 +-- Cutout/Renderer/Renderer.cs | 47 ++- Cutout/TemplateAttributeParts.cs | 3 +- README.md | 37 +-- 15 files changed, 849 insertions(+), 434 deletions(-) create mode 100644 Cutout/Lexer/Lexer.ApplyWhitespaceSuppression.cs create mode 100644 Cutout/Lexer/Lexer.Context.cs create mode 100644 Cutout/Parser/Parser.ParseBlock.cs delete mode 100644 Cutout/Parser/Parser.ParseCodeBlock.cs diff --git a/Cutout.Tests/LexerTests.cs b/Cutout.Tests/LexerTests.cs index 573912e..3bf276b 100644 --- a/Cutout.Tests/LexerTests.cs +++ b/Cutout.Tests/LexerTests.cs @@ -44,13 +44,13 @@ public void Case1() [Fact(DisplayName = "Code blocks can be tokenized")] public void Case2() { - const string text = "{{ code block }}"; + const string text = "{{ render block }}"; var tokens = Lexer.Tokenize(text); Assert.Collection( tokens, token => { - Assert.Equal(TokenType.CodeEnter, token.Type); + Assert.Equal(TokenType.RenderEnter, token.Type); Assert.Equal(0, token.Start.Offset); Assert.Equal(1, token.End.Offset); Assert.Equal("{{", token.ToSpan(text).ToString()); @@ -66,36 +66,36 @@ public void Case2() { Assert.Equal(TokenType.Raw, token.Type); Assert.Equal(3, token.Start.Offset); - Assert.Equal(6, token.End.Offset); - Assert.Equal("code", token.ToSpan(text).ToString()); + Assert.Equal(8, token.End.Offset); + Assert.Equal("render", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(7, token.Start.Offset); - Assert.Equal(7, token.End.Offset); + Assert.Equal(9, token.Start.Offset); + Assert.Equal(9, token.End.Offset); Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Raw, token.Type); - Assert.Equal(8, token.Start.Offset); - Assert.Equal(12, token.End.Offset); + Assert.Equal(10, token.Start.Offset); + Assert.Equal(14, token.End.Offset); Assert.Equal("block", token.ToSpan(text).ToString()); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(13, token.Start.Offset); - Assert.Equal(13, token.End.Offset); + Assert.Equal(15, token.Start.Offset); + Assert.Equal(15, token.End.Offset); Assert.Equal(" ", token.ToSpan(text).ToString()); }, token => { - Assert.Equal(TokenType.CodeExit, token.Type); - Assert.Equal(14, token.Start.Offset); - Assert.Equal(15, token.End.Offset); + Assert.Equal(TokenType.RenderExit, token.Type); + Assert.Equal(16, token.Start.Offset); + Assert.Equal(17, token.End.Offset); Assert.Equal("}}", token.ToSpan(text).ToString()); }, token => @@ -223,7 +223,7 @@ public void Case4() }, token => { - Assert.Equal(TokenType.CodeEnter, token.Type); + Assert.Equal(TokenType.RenderEnter, token.Type); Assert.Equal(8, token.Start.Offset); Assert.Equal(9, token.End.Offset); Assert.Equal("{{", token.ToSpan(text).ToString()); @@ -265,7 +265,7 @@ public void Case4() }, token => { - Assert.Equal(TokenType.CodeExit, token.Type); + Assert.Equal(TokenType.RenderExit, token.Type); Assert.Equal(22, token.Start.Offset); Assert.Equal(23, token.End.Offset); Assert.Equal("}}", token.ToSpan(text).ToString()); @@ -349,7 +349,7 @@ public void Case4() }, token => { - Assert.Equal(TokenType.CodeEnter, token.Type); + Assert.Equal(TokenType.RenderEnter, token.Type); Assert.Equal(46, token.Start.Offset); Assert.Equal(47, token.End.Offset); Assert.Equal("{{", token.ToSpan(text).ToString()); @@ -391,7 +391,7 @@ public void Case4() }, token => { - Assert.Equal(TokenType.CodeExit, token.Type); + Assert.Equal(TokenType.RenderExit, token.Type); Assert.Equal(59, token.Start.Offset); Assert.Equal(60, token.End.Offset); Assert.Equal("}}", token.ToSpan(text).ToString()); @@ -404,4 +404,83 @@ public void Case4() } ); } + + [Fact(DisplayName = "Render with whitespace suppression can be tokenized")] + public void Case5() + { + const string text = "{{- render block -}}"; + var tokens = Lexer.Tokenize(text); + Assert.Collection( + tokens, + tokens => + { + Assert.Equal(TokenType.RenderSuppressWsEnter, tokens.Type); + Assert.Equal("{{-", tokens.ToSpan(text).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(" ", token.ToSpan(text).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal("render", token.ToSpan(text).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(" ", token.ToSpan(text).ToString()); + }, + token => + { + Assert.Equal(TokenType.Raw, token.Type); + Assert.Equal("block", token.ToSpan(text).ToString()); + }, + token => + { + Assert.Equal(TokenType.Whitespace, token.Type); + Assert.Equal(" ", token.ToSpan(text).ToString()); + }, + token => + { + Assert.Equal(TokenType.RenderSuppressWsExit, token.Type); + Assert.Equal("-}}", token.ToSpan(text).ToString()); + }, + token => + { + Assert.Equal(TokenType.Eof, token.Type); + } + ); + } + + [Fact(DisplayName = "Render with whitespace suppression and newlines can be tokenized")] + public void Case6() + { + const string text = "{{ render block -}}\n{% more code -%} \n"; + var tokens = Lexer.Tokenize(text); + Assert.Equal(text, tokens.ToString(text)); + var withWsSuppression = Lexer.ApplyWhitespaceSuppression(tokens); + Assert.Equal("{{ render block -}}{% more code -%}", withWsSuppression.ToString(text)); + } + + [Fact( + DisplayName = "Render with whitespace suppression and newlines can be tokenized (leading and trailing spaces)" + )] + public void Case7() + { + const string text = " \n {{- render block -}}\n {%- more code -%} \n "; + var tokens = Lexer.Tokenize(text); + Assert.Equal(text, tokens.ToString(text)); + var withWsSuppression = Lexer.ApplyWhitespaceSuppression(tokens); + Assert.Equal(" {{- render block -}}{%- more code -%} ", withWsSuppression.ToString(text)); + } + + [Fact(DisplayName = "Code and render blocks can be mixed in a single text")] + public void Case8() + { + const string text = "This is {{ render block }}\n{% code block %} "; + var tokens = Lexer.Tokenize(text); + Assert.Equal(text, tokens.ToString(text)); + } } diff --git a/Cutout.Tests/ParserTests.cs b/Cutout.Tests/ParserTests.cs index 78a49fd..7af13ea 100644 --- a/Cutout.Tests/ParserTests.cs +++ b/Cutout.Tests/ParserTests.cs @@ -74,17 +74,17 @@ public void Case3() item => { var rawText = Assert.IsType(item); - Assert.Equal("raw ", rawText.Value.ToSpan(template)); + Assert.Equal("raw ", rawText.Value.ToString(template)); }, item => { var renderableExpression = Assert.IsType(item); - Assert.Equal(" code ", renderableExpression.Value.ToSpan(template)); + Assert.Equal(" code ", renderableExpression.Value.ToString(template)); }, item => { var rawText = Assert.IsType(item); - Assert.Equal(" string", rawText.Value.ToSpan(template)); + Assert.Equal(" string", rawText.Value.ToString(template)); } ); } @@ -99,7 +99,7 @@ public void Case4() Assert.Empty(result); } - [Fact(DisplayName = "Code blocks must have balanced braces")] + [Fact(DisplayName = "Render blocks must have balanced braces")] public void Case5() { const string template = "{{ code"; @@ -112,6 +112,19 @@ public void Case5() Assert.Equal("code", exception.Token.ToSpan(template).ToString()); } + [Fact(DisplayName = "Code blocks must have balanced braces")] + public void Case5b() + { + const string template = "{% code"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Raw): Code exit token not found (value: 'code')", + exception.Message + ); + Assert.Equal("code", exception.Token.ToSpan(template).ToString()); + } + [Fact(DisplayName = "Code blocks cannot be empty")] public void Case6() { @@ -126,24 +139,70 @@ public void Case6() Assert.Equal(" ", exception.Value); } + [Fact(DisplayName = "Render blocks cannot be empty")] + public void Case6b() + { + const string template = "{%- -%}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:4 (Whitespace): Code block is empty (value: ' ')", + exception.Message + ); + Assert.Equal(TokenType.Whitespace, exception.Token.Type); + Assert.Equal(" ", exception.Value); + } + + [Fact( + DisplayName = "Blocks must close with the token type they opened with (Code open, render close)" + )] + public void Case6c() + { + const string template = "{% if test }}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:12 (RenderExit): Render block exit token cannot be used in code blocks (value: '}}')", + exception.Message + ); + Assert.Equal(TokenType.RenderExit, exception.Token.Type); + Assert.Equal("}}", exception.Value); + } + + [Fact( + DisplayName = "Blocks must close with the token type they opened with (Render open, code close)" + )] + public void Case6d() + { + const string template = "{{ if test %}"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:12 (CodeExit): Code block exit token cannot be used in render blocks (value: '%}')", + exception.Message + ); + Assert.Equal(TokenType.CodeExit, exception.Token.Type); + Assert.Equal("%}", exception.Value); + } + [Fact(DisplayName = "Nested code blocks are not allowed")] public void Case7() { - const string template = "{{ {{ nested }} }}"; + const string template = "{{ {% nested %} }}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (CodeEnter): Nested code blocks are not allowed (value: '{{')", + "Parse error at 1:4 (CodeEnter): Nested code or render blocks are not allowed (value: '{%')", exception.Message ); Assert.Equal(TokenType.CodeEnter, exception.Token.Type); - Assert.Equal("{{", exception.Value); + Assert.Equal("{%", exception.Value); } [Fact(DisplayName = "A break statement can be parsed")] public void Case8() { - const string template = "{{ break }}"; + const string template = "{% break %}"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); @@ -154,7 +213,7 @@ public void Case8() [Fact(DisplayName = "A continue statement can be parsed")] public void Case9() { - const string template = "{{ continue }}"; + const string template = "{% continue %}"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); @@ -165,7 +224,7 @@ public void Case9() [Fact(DisplayName = "A return statement can be parsed")] public void Case10() { - const string template = "{{ return }}"; + const string template = "{% return %}"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); @@ -176,7 +235,7 @@ public void Case10() [Fact(DisplayName = "A var statement can be parsed")] public void Case11() { - const string template = "{{ var x = 42 }}"; + const string template = "{% var x = 42 %}"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); @@ -225,11 +284,11 @@ public void Case11() [Fact(DisplayName = "A var statement without an expression throws an error")] public void Case12() { - const string template = "{{ var }}"; + const string template = "{% var %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ var }} declaration requires an assignment expression (value: 'var')", + "Parse error at 1:4 (Raw): {% var %} declaration requires an assignment expression (value: 'var')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -239,7 +298,7 @@ public void Case12() [Fact(DisplayName = "A call statement can be parsed")] public void Case13() { - const string template = "{{ call function(arg1, arg2) }}"; + const string template = "{% call function(arg1, arg2) %}"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); @@ -262,7 +321,7 @@ public void Case13() [Fact(DisplayName = "A call statement can be parsed (no arguments)")] public void Case13b() { - const string template = "{{ call function() }}"; + const string template = "{% call function() %}"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); @@ -275,11 +334,11 @@ public void Case13b() [Fact(DisplayName = "A call statement function part throws an error")] public void Case14() { - const string template = "{{ call }}"; + const string template = "{% call %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ call }} statement requires parameters (value: 'call')", + "Parse error at 1:4 (Raw): {% call %} statement requires parameters (value: 'call')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -289,11 +348,11 @@ public void Case14() [Fact(DisplayName = "A call statement without parentheses throws an error")] public void Case15() { - const string template = "{{ call function }}"; + const string template = "{% call function %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ call }} statement requires a function name and () with optional parameters (value: 'call')", + "Parse error at 1:4 (Raw): {% call %} statement requires a function name and () with optional parameters (value: 'call')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -303,11 +362,11 @@ public void Case15() [Fact(DisplayName = "A call statement without parentheses throws an error (first only)")] public void Case15b() { - const string template = "{{ call function( }}"; + const string template = "{% call function( %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ call }} statement requires a function name and () with optional parameters (value: 'call')", + "Parse error at 1:4 (Raw): {% call %} statement requires a function name and () with optional parameters (value: 'call')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -317,11 +376,11 @@ public void Case15b() [Fact(DisplayName = "A call statement without parentheses throws an error (last only)")] public void Case15c() { - const string template = "{{ call function) }}"; + const string template = "{% call function) %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ call }} statement requires a function name and () with optional parameters (value: 'call')", + "Parse error at 1:4 (Raw): {% call %} statement requires a function name and () with optional parameters (value: 'call')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -331,11 +390,11 @@ public void Case15c() [Fact(DisplayName = "A call statement with only parentheses throws an error")] public void Case16() { - const string template = "{{ call () }}"; + const string template = "{% call () %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ call }} statement requires a function name and () with optional parameters (value: 'call')", + "Parse error at 1:4 (Raw): {% call %} statement requires a function name and () with optional parameters (value: 'call')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -345,7 +404,7 @@ public void Case16() [Fact(DisplayName = "A while statement can be parsed")] public void Case17() { - const string template = "{{ while condition }} some code {{ end }}"; + const string template = "{% while condition %} some code {% end %}"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); @@ -405,11 +464,11 @@ public void Case17() [Fact(DisplayName = "A while statement without a condition throws an error")] public void Case18() { - const string template = "{{ while }}"; + const string template = "{% while %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ while }} statement requires a condition (value: 'while')", + "Parse error at 1:4 (Raw): {% while %} statement requires a condition (value: 'while')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -419,11 +478,11 @@ public void Case18() [Fact(DisplayName = "A while statement without an end token throws an error")] public void Case19() { - const string template = "{{ while condition }} some code"; + const string template = "{% while condition %} some code"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at end of file: Unexpected end of file, expected a {{ end }} block", + "Parse error at end of file: Unexpected end of file, expected a {% end %} block", exception.Message ); Assert.Equal(TokenType.Eof, exception.Token.Type); @@ -433,59 +492,59 @@ public void Case19() [Fact(DisplayName = "A for statement can be parsed")] public void Case20() { - const string template = "{{ for i = 0; i < items.Length; i++ }} some code {{ end }}"; + const string template = "{% for i = 0; i < items.Length; i++ %} some code {% end %}"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); var item = Assert.Single(result); var forStatement = Assert.IsType(item); - Assert.Equal(" i = 0; i < items.Length; i++ ", forStatement.Condition.ToSpan(template)); + Assert.Equal(" i = 0; i < items.Length; i++ ", forStatement.Condition.ToString(template)); var expression = Assert.Single(forStatement.Expressions); var rawText = Assert.IsType(expression); - Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + Assert.Equal(" some code ", rawText.Value.ToString(template)); } [Fact(DisplayName = "A foreach statement can be parsed")] public void Case21() { - const string template = "{{ foreach item in items }} some code {{ end }}"; + const string template = "{% foreach item in items %} some code {% end %}"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); var item = Assert.Single(result); var foreachStatement = Assert.IsType(item); - Assert.Equal(" item in items ", foreachStatement.Condition.ToSpan(template)); + Assert.Equal(" item in items ", foreachStatement.Condition.ToString(template)); var expression = Assert.Single(foreachStatement.Expressions); var rawText = Assert.IsType(expression); - Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + Assert.Equal(" some code ", rawText.Value.ToString(template)); } [Fact(DisplayName = "An if statement can be parsed")] public void Case22() { - const string template = "{{ if condition }} some code {{ end }}"; + const string template = "{% if condition %} some code {% end %}"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); var item = Assert.Single(result); var ifStatement = Assert.IsType(item); - Assert.Equal(" condition ", ifStatement.Condition.ToSpan(template)); + Assert.Equal(" condition ", ifStatement.Condition.ToString(template)); var expression = Assert.Single(ifStatement.Expressions); var rawText = Assert.IsType(expression); - Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + Assert.Equal(" some code ", rawText.Value.ToString(template)); } [Fact(DisplayName = "An if statement without a condition throws an error")] public void Case23() { - const string template = "{{ if }}"; + const string template = "{% if %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ if }} statement requires a condition (value: 'if')", + "Parse error at 1:4 (Raw): {% if %} statement requires a condition (value: 'if')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -495,11 +554,11 @@ public void Case23() [Fact(DisplayName = "An if statement without an end token throws an error")] public void Case24() { - const string template = "{{ if condition }} some code"; + const string template = "{% if condition %} some code"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at end of file: Unexpected end of file, expected a {{ end }} block", + "Parse error at end of file: Unexpected end of file, expected a {% end %} block", exception.Message ); Assert.Equal(TokenType.Eof, exception.Token.Type); @@ -509,11 +568,11 @@ public void Case24() [Fact(DisplayName = "An else without an if statement throws an error")] public void Case25() { - const string template = "{{ else }} some code {{ end }}"; + const string template = "{% else %} some code {% end %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ else }} found but not expected (value: 'else')", + "Parse error at 1:4 (Raw): {% else %} found but not expected (value: 'else')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -523,11 +582,11 @@ public void Case25() [Fact(DisplayName = "An else if without an if statement throws an error")] public void Case26() { - const string template = "{{ elseif condition }} some code {{ end }}"; + const string template = "{% elseif condition %} some code {% end %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ elseif }} found but not expected (value: 'elseif')", + "Parse error at 1:4 (Raw): {% elseif %} found but not expected (value: 'elseif')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -537,33 +596,33 @@ public void Case26() [Fact(DisplayName = "An else statement can be parsed")] public void Case27() { - const string template = "{{ if condition }} some code {{ else }} other code {{ end }}"; + const string template = "{% if condition %} some code {% else %} other code {% end %}"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); var single = Assert.Single(result); var ifStatement = Assert.IsType(single); - Assert.Equal(" condition ", ifStatement.Condition.ToSpan(template)); + Assert.Equal(" condition ", ifStatement.Condition.ToString(template)); var expression = Assert.Single(ifStatement.Expressions); var rawText = Assert.IsType(expression); - Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + Assert.Equal(" some code ", rawText.Value.ToString(template)); Assert.NotNull(ifStatement.Else); expression = Assert.Single(ifStatement.Else.Expressions); rawText = Assert.IsType(expression); - Assert.Equal(" other code ", rawText.Value.ToSpan(template)); + Assert.Equal(" other code ", rawText.Value.ToString(template)); } [Fact(DisplayName = "An else after an else statement throws an error")] public void Case28() { const string template = - "{{ if condition }} some code {{ else }} other code {{ else }} more code {{ end }}"; + "{% if condition %} some code {% else %} other code {% else %} more code {% end %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:55 (Raw): Only one {{ else }} is allowed within an {{ if }} statement (value: 'else')", + "Parse error at 1:55 (Raw): Only one {% else %} is allowed within an {% if %} statement (value: 'else')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -574,34 +633,34 @@ public void Case28() public void Case29() { const string template = - "{{ if condition }} some code {{ elseif otherCondition }} other code {{ end }}"; + "{% if condition %} some code {% elseif otherCondition %} other code {% end %}"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); var single = Assert.Single(result); var ifStatement = Assert.IsType(single); - Assert.Equal(" condition ", ifStatement.Condition.ToSpan(template)); + Assert.Equal(" condition ", ifStatement.Condition.ToString(template)); var expression = Assert.Single(ifStatement.Expressions); var rawText = Assert.IsType(expression); - Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + Assert.Equal(" some code ", rawText.Value.ToString(template)); Assert.NotNull(ifStatement.ElseIfs); single = Assert.Single(ifStatement.ElseIfs); var elseIfStatement = Assert.IsType(single); - Assert.Equal(" otherCondition ", elseIfStatement.Condition.ToSpan(template)); + Assert.Equal(" otherCondition ", elseIfStatement.Condition.ToString(template)); expression = Assert.Single(elseIfStatement.Expressions); rawText = Assert.IsType(expression); - Assert.Equal(" other code ", rawText.Value.ToSpan(template)); + Assert.Equal(" other code ", rawText.Value.ToString(template)); } [Fact(DisplayName = "An else if after an else statement throws an error")] public void Case30() { const string template = - "{{ if condition }} some code {{ else }} other code {{ elseif anotherCondition }} more code {{ end }}"; + "{% if condition %} some code {% else %} other code {% elseif anotherCondition %} more code {% end %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:55 (Raw): Cannot have {{ elseif }} after {{ else }} in an {{ if }} statement (value: 'elseif')", + "Parse error at 1:55 (Raw): Cannot have {% elseif %} after {% else %} in an {% if %} statement (value: 'elseif')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -612,37 +671,37 @@ public void Case30() public void Case32() { const string template = - "{{ if condition }} some code {{ elseif otherCondition }} other code {{ else }} final code {{ end }}"; + "{% if condition %} some code {% elseif otherCondition %} other code {% else %} final code {% end %}"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); var single = Assert.Single(result); var ifStatement = Assert.IsType(single); - Assert.Equal(" condition ", ifStatement.Condition.ToSpan(template)); + Assert.Equal(" condition ", ifStatement.Condition.ToString(template)); var expression = Assert.Single(ifStatement.Expressions); var rawText = Assert.IsType(expression); - Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + Assert.Equal(" some code ", rawText.Value.ToString(template)); Assert.NotNull(ifStatement.ElseIfs); single = Assert.Single(ifStatement.ElseIfs); var elseIfStatement = Assert.IsType(single); - Assert.Equal(" otherCondition ", elseIfStatement.Condition.ToSpan(template)); + Assert.Equal(" otherCondition ", elseIfStatement.Condition.ToString(template)); expression = Assert.Single(elseIfStatement.Expressions); rawText = Assert.IsType(expression); - Assert.Equal(" other code ", rawText.Value.ToSpan(template)); + Assert.Equal(" other code ", rawText.Value.ToString(template)); Assert.NotNull(ifStatement.Else); expression = Assert.Single(ifStatement.Else.Expressions); rawText = Assert.IsType(expression); - Assert.Equal(" final code ", rawText.Value.ToSpan(template)); + Assert.Equal(" final code ", rawText.Value.ToString(template)); } [Fact(DisplayName = "A end statement without a block throws an error")] public void Case33() { - const string template = "{{ end }}"; + const string template = "{% end %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ end }} found but not expected (value: 'end')", + "Parse error at 1:4 (Raw): {% end %} found but not expected (value: 'end')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -652,11 +711,11 @@ public void Case33() [Fact(DisplayName = "A end statement with extra tokens throws an error")] public void Case34() { - const string template = "{{ end extra }}"; + const string template = "{% end extra %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:4 (Raw): {{ end }} statement should only contain the identifier (value: 'end')", + "Parse error at 1:4 (Raw): {% end %} statement should only contain the identifier (value: 'end')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -666,11 +725,11 @@ public void Case34() [Fact(DisplayName = "A else statement with a condition throws an error")] public void Case36() { - const string template = "{{ if test }} something {{ else condition }} some code {{ end }}"; + const string template = "{% if test %} something {% else condition %} some code {% end %}"; var tokens = Lexer.Tokenize(template); var exception = Assert.Throws(() => Parser.Parse(tokens, template)); Assert.Equal( - "Parse error at 1:28 (Raw): {{ else }} statement should only contain the identifier (value: 'else')", + "Parse error at 1:28 (Raw): {% else %} statement should only contain the identifier (value: 'else')", exception.Message ); Assert.Equal(TokenType.Raw, exception.Token.Type); @@ -681,13 +740,13 @@ public void Case36() public void Case37() { const string template = - "{{ if condition1 }} some code {{ if condition2 }} nested code {{ end }}{{ end }} some more code"; + "{% if condition1 %} some code {% if condition2 %} nested code {% end %}{% end %} some more code"; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); Assert.Equal(2, result.Count); var ifStatement = Assert.IsType(result[0]); - Assert.Equal(" condition1 ", ifStatement.Condition.ToSpan(template)); + Assert.Equal(" condition1 ", ifStatement.Condition.ToString(template)); Assert.Equal(2, ifStatement.Expressions.Count); Assert.Collection( @@ -695,60 +754,60 @@ public void Case37() expression => { var rawText = Assert.IsType(expression); - Assert.Equal(" some code ", rawText.Value.ToSpan(template)); + Assert.Equal(" some code ", rawText.Value.ToString(template)); }, expression => { var nestedIfStatement = Assert.IsType(expression); - Assert.Equal(" condition2 ", nestedIfStatement.Condition.ToSpan(template)); + Assert.Equal(" condition2 ", nestedIfStatement.Condition.ToString(template)); Assert.Single(nestedIfStatement.Expressions); var nestedRawText = Assert.IsType(nestedIfStatement.Expressions[0]); - Assert.Equal(" nested code ", nestedRawText.Value.ToSpan(template)); + Assert.Equal(" nested code ", nestedRawText.Value.ToString(template)); } ); var rawText = Assert.IsType(result[1]); - Assert.Equal(" some more code", rawText.Value.ToSpan(template)); + Assert.Equal(" some more code", rawText.Value.ToString(template)); } [Fact(DisplayName = "A if else statement can be nested inside an if else statement")] public void Case39() { const string template = """ - {{ if condition1 }} + {% if condition1 %} nested - {{ if condition2 }} + {% if condition2 %} test - {{ else }} + {% else %} other - {{ end }} + {% end %} code - {{ else }} + {% else %} final - {{ if test }} + {% if test %} test - {{ else }} + {% else %} other - {{ end }} + {% end %} code - {{ end }} + {% end %} """; var tokens = Lexer.Tokenize(template); var result = Parser.Parse(tokens, template); var single = Assert.Single(result); var ifStatement = Assert.IsType(single); - Assert.Equal(" condition1 ", ifStatement.Condition.ToSpan(template)); + Assert.Equal(" condition1 ", ifStatement.Condition.ToString(template)); Assert.Equal(3, ifStatement.Expressions.Count); var nestedIf = Assert.IsType(ifStatement.Expressions[1]); - Assert.Equal(" condition2 ", nestedIf.Condition.ToSpan(template)); + Assert.Equal(" condition2 ", nestedIf.Condition.ToString(template)); Assert.Single(nestedIf.Expressions); Assert.NotNull(nestedIf.Else); Assert.Single(nestedIf.Else.Expressions); Assert.NotNull(ifStatement.Else); Assert.Equal(3, ifStatement.Else.Expressions.Count); var elseNestedIf = Assert.IsType(ifStatement.Else.Expressions[1]); - Assert.Equal(" test ", elseNestedIf.Condition.ToSpan(template)); + Assert.Equal(" test ", elseNestedIf.Condition.ToString(template)); Assert.Single(elseNestedIf.Expressions); Assert.NotNull(elseNestedIf.Else); Assert.Single(elseNestedIf.Else.Expressions); diff --git a/Cutout/Extensions/TokenListExtensions.cs b/Cutout/Extensions/TokenListExtensions.cs index f5ea819..903214a 100644 --- a/Cutout/Extensions/TokenListExtensions.cs +++ b/Cutout/Extensions/TokenListExtensions.cs @@ -1,16 +1,29 @@ -namespace Cutout; +using System.Text; + +namespace Cutout; internal static class TokenListExtensions { - internal static ReadOnlySpan ToSpan(this TokenList list, string template) + internal static string ToString(this TokenList list, string template) { if (list.Count == 0) { - return ReadOnlySpan.Empty; + return string.Empty; } - var start = list[0]; - var end = list[list.Count - 1]; - return start.ToSpan(template, end); + var builder = new StringBuilder(); + foreach (var token in list) + { + if (token.Type == TokenType.Eof) + { + break; + } + + foreach (var c in token.ToSpan(template)) + { + builder.Append(c); + } + } + return builder.ToString(); } } diff --git a/Cutout/Lexer/Lexer.ApplyWhitespaceSuppression.cs b/Cutout/Lexer/Lexer.ApplyWhitespaceSuppression.cs new file mode 100644 index 0000000..23112e1 --- /dev/null +++ b/Cutout/Lexer/Lexer.ApplyWhitespaceSuppression.cs @@ -0,0 +1,64 @@ +namespace Cutout; + +internal static partial class Lexer +{ + internal static TokenList ApplyWhitespaceSuppression(TokenList tokens) + { + var result = new TokenList(); + var i = 0; + while (i < tokens.Count) + { + var token = tokens[i]; + switch (token.Type) + { + case TokenType.RenderSuppressWsEnter: + case TokenType.CodeSuppressWsEnter: + { + RemoveLeadingWhitespaceAndNewline(result); + result.Add(token); + i++; + break; + } + case TokenType.RenderSuppressWsExit: + case TokenType.CodeSuppressWsExit: + { + result.Add(token); + i = SkipTrailingWhitespaceAndNewline(tokens, i + 1); + break; + } + default: + result.Add(token); + i++; + break; + } + } + return result; + } + + private static void RemoveLeadingWhitespaceAndNewline(TokenList result) + { + var index = result.Count - 1; + if (index >= 0 && result[index].Type == TokenType.Whitespace) + { + result.RemoveAt(index--); + } + if (index >= 0 && result[index].Type == TokenType.Newline) + { + result.RemoveAt(index); + } + } + + private static int SkipTrailingWhitespaceAndNewline(TokenList tokens, int startIndex) + { + var index = startIndex; + if (index < tokens.Count && tokens[index].Type == TokenType.Whitespace) + { + index++; + } + if (index < tokens.Count && tokens[index].Type == TokenType.Newline) + { + index++; + } + return index; + } +} diff --git a/Cutout/Lexer/Lexer.Context.cs b/Cutout/Lexer/Lexer.Context.cs new file mode 100644 index 0000000..7ae0914 --- /dev/null +++ b/Cutout/Lexer/Lexer.Context.cs @@ -0,0 +1,90 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; + +namespace Cutout; + +internal static partial class Lexer +{ + public sealed class Context : IEnumerator + { + public int Index { get; private set; } + + public int Column { get; private set; } + + public bool TryAdvance(int count) + { + if (Index + count >= Template.Length) + { + return false; + } + + Index += count; + Column += count; + return true; + } + + public int Line { get; private set; } + + public void AdvanceNewline(bool windowsNewline = false) + { + if (windowsNewline) + { + Index++; + } + + Line++; + Column = 0; + } + + public string Template { get; private set; } = null!; + + public ReadOnlySpan Peek(int count) + { + return Index + count >= Template.Length + ? Template.AsSpan(Index) + : Template.AsSpan(Index, count); + } + + [ExcludeFromCodeCoverage] + public void Dispose() + { + Reset(); + } + + public bool MoveNext() + { + return TryAdvance(count: 1); + } + + public void Reset() + { + Index = -1; + Column = 0; + Line = 1; + } + + public void Reset(string template) + { + Template = template; + Reset(); + } + + public char Current => Template[Index]; + + [ExcludeFromCodeCoverage] + object IEnumerator.Current => Current; + + public CharPosition CurrentPosition => new(Line, Column, Index); + + public Token Token(TokenType type, int count) + { + var start = CurrentPosition; + var end = new CharPosition( + start.Line, + start.Column + count, + CurrentPosition.Offset + count + ); + return new(start, end, type); + } + } +} diff --git a/Cutout/Lexer/Lexer.cs b/Cutout/Lexer/Lexer.cs index 90da6b8..29999cb 100644 --- a/Cutout/Lexer/Lexer.cs +++ b/Cutout/Lexer/Lexer.cs @@ -1,189 +1,164 @@ namespace Cutout; -internal static class Lexer +internal static partial class Lexer { - internal static TokenList Tokenize(in ReadOnlySpan text) + [ThreadStatic] + private static Context? _context; + + internal static TokenList Tokenize(string text) { var tokens = new TokenList(); - var i = 0; - var line = 1; - var column = 1; - var length = text.Length; - - while ( - i < length - && ( - TryProcessNewline(text, ref i, ref line, ref column, length, tokens) - || TryProcessCodeDelimiters(text, ref i, line, ref column, length, tokens) - || TryProcessWhitespace(text, ref i, line, ref column, length, tokens) - || TryProcessRawText(text, ref i, line, ref column, length, tokens) - ) - ) - { // collect tokens + _context ??= new Context(); + _context.Reset(text); + + var collectCount = 0; + var isWhitespace = false; + CharPosition? start = null; + while (_context.MoveNext()) + { + if (TryProcessNewline(_context, out var newLineToken)) + { + FlushAndAdd(newLineToken); + } + else if (TryProcessCodeDelimiters(_context, out var codeToken)) + { + FlushAndAdd(codeToken); + } + else + { + if (isWhitespace ^ char.IsWhiteSpace(_context.Current)) + { + FlushCollected(); + isWhitespace = !isWhitespace; + } + + start ??= _context.CurrentPosition; + collectCount++; + } } - tokens.Add(Token.Eof); + FlushAndAdd(Token.Eof); return tokens; - } - private static bool TryProcessNewline( - ReadOnlySpan text, - ref int i, - ref int line, - ref int column, - int length, - TokenList tokens - ) - { - var c = text[i]; - var currentPosition = new CharPosition(line, column, i); + void FlushCollected() + { + if (collectCount <= 0) + { + start = null; + return; + } + + tokens.Add( + start!.Value.ToToken( + isWhitespace ? TokenType.Whitespace : TokenType.Raw, + collectCount + ) + ); + start = null; + collectCount = 0; + } - switch (c) + void FlushAndAdd(Token token) { - case '\r' when i + 1 < length && text[i + 1] == '\n': - tokens.Add( - new Token( - currentPosition, - new CharPosition(line, column + 1, i + 1), - TokenType.Newline - ) - ); - i += 2; - line++; - column = 1; - return true; - case '\n': - tokens.Add(new Token(currentPosition, currentPosition, TokenType.Newline)); - i++; - line++; - column = 1; - return true; - default: - return false; + FlushCollected(); + tokens.Add(token); } } - private static bool TryProcessCodeDelimiters( - ReadOnlySpan text, - ref int i, - int line, - ref int column, - int length, - TokenList tokens - ) + private static readonly char[] WindowsNewline = ['\r', '\n']; + private static readonly char[] UnixNewline = ['\n']; + + private static bool TryProcessNewline(Context context, out Token token) { - var c = text[i]; - var currentPosition = new CharPosition(line, column, i); + if (context.Peek(2).SequenceEqual(WindowsNewline)) + { + token = context.Token(TokenType.Newline, 1); + context.AdvanceNewline(windowsNewline: true); + return true; + } - switch (c) + if (context.Peek(1).SequenceEqual(UnixNewline)) { - case '{' when i + 1 < length && text[i + 1] == '{': - tokens.Add( - new Token( - currentPosition, - new CharPosition(line, column + 1, i + 1), - TokenType.CodeEnter - ) - ); - i += 2; - column += 2; - return true; - case '}' when i + 1 < length && text[i + 1] == '}': - tokens.Add( - new Token( - currentPosition, - new CharPosition(line, column + 1, i + 1), - TokenType.CodeExit - ) - ); - i += 2; - column += 2; - return true; - default: - return false; + token = context.Token(TokenType.Newline, 0); + context.AdvanceNewline(windowsNewline: false); + return true; } + + token = default; + return false; } - private static bool TryProcessWhitespace( - ReadOnlySpan text, - ref int i, - int line, - ref int column, - int length, - TokenList tokens - ) - { - var c = text[i]; + private static readonly char[] RenderSuppressWsEnter = ['{', '{', '-']; + private static readonly char[] RenderEnter = ['{', '{']; + private static readonly char[] RenderSuppressWsExit = ['-', '}', '}']; + private static readonly char[] RenderExit = ['}', '}']; + private static readonly char[] CodeSuppressWsEnter = ['{', '%', '-']; + private static readonly char[] CodeEnter = ['{', '%']; + private static readonly char[] CodeSuppressWsExit = ['-', '%', '}']; + private static readonly char[] CodeExit = ['%', '}']; + private static bool TryProcessCodeDelimiters(Context context, out Token token) + { if ( - !char.IsWhiteSpace(c) - || c == '\n' - || (c == '\r' && i + 1 < length && text[i + 1] == '\n') + context.Current != '{' + && context.Current != '%' + && context.Current != '-' + && context.Current != '}' ) { + token = default; return false; } - var start = i; - var columnStart = column; + if (context.Peek(3).SequenceEqual(RenderSuppressWsEnter)) + { + token = context.Token(TokenType.RenderSuppressWsEnter, 2); + return context.TryAdvance(count: 2); + } + + if (context.Peek(2).SequenceEqual(RenderEnter)) + { + token = context.Token(TokenType.RenderEnter, 1); + return context.TryAdvance(count: 1); + } - while ( - i < length - && char.IsWhiteSpace(text[i]) - && text[i] != '\n' - && !(text[i] == '\r' && i + 1 < length && text[i + 1] == '\n') - ) + if (context.Peek(3).SequenceEqual(RenderSuppressWsExit)) { - i++; - column++; + token = context.Token(TokenType.RenderSuppressWsExit, 2); + return context.TryAdvance(count: 2); } - tokens.Add( - new Token( - new CharPosition(line, columnStart, start), - new CharPosition(line, column - 1, i - 1), - TokenType.Whitespace - ) - ); + if (context.Peek(2).SequenceEqual(RenderExit)) + { + token = context.Token(TokenType.RenderExit, 1); + return context.TryAdvance(count: 1); + } - return true; - } + if (context.Peek(3).SequenceEqual(CodeSuppressWsEnter)) + { + token = context.Token(TokenType.CodeSuppressWsEnter, 2); + return context.TryAdvance(count: 2); + } - private static bool TryProcessRawText( - ReadOnlySpan text, - ref int i, - int line, - ref int column, - int length, - TokenList tokens - ) - { - var start = i; - var columnStart = column; - - while ( - i < length - && !( - char.IsWhiteSpace(text[i]) - || (text[i] == '{' && i + 1 < length && text[i + 1] == '{') - || (text[i] == '}' && i + 1 < length && text[i + 1] == '}') - || text[i] == '\n' - || (text[i] == '\r' && i + 1 < length && text[i + 1] == '\n') - ) - ) + if (context.Peek(2).SequenceEqual(CodeEnter)) + { + token = context.Token(TokenType.CodeEnter, 1); + return context.TryAdvance(count: 1); + } + + if (context.Peek(3).SequenceEqual(CodeSuppressWsExit)) { - i++; - column++; + token = context.Token(TokenType.CodeSuppressWsExit, 2); + return context.TryAdvance(count: 2); } - tokens.Add( - new Token( - new CharPosition(line, columnStart, start), - new CharPosition(line, column - 1, i - 1), - TokenType.Raw - ) - ); + if (context.Peek(2).SequenceEqual(CodeExit)) + { + token = context.Token(TokenType.CodeExit, 1); + return context.TryAdvance(count: 1); + } - return true; + token = default; + return false; } } diff --git a/Cutout/Lexer/Token.cs b/Cutout/Lexer/Token.cs index bbc7993..a387b3c 100644 --- a/Cutout/Lexer/Token.cs +++ b/Cutout/Lexer/Token.cs @@ -26,20 +26,59 @@ internal enum TokenType : byte Raw, /// - /// Start of a code block "{{" + /// Start of a render block "{{" + /// + RenderEnter, + + /// + /// Start of a render block "{{-" such that any leading whitespace and newline are suppressed + /// + RenderSuppressWsEnter, + + /// + /// End of a render block "}}" + /// + RenderExit, + + /// + /// End of a render block "-}}" such that any trailing whitespace and newline are suppressed + /// + RenderSuppressWsExit, + + /// + /// Start of a code block "{%" /// CodeEnter, /// - /// End of a code block "}}" + /// Start of a code block "{%-" such that any leading whitespace and newline are suppressed + /// + CodeSuppressWsEnter, + + /// + /// End of a code block "%}" /// CodeExit, + + /// + /// End of a code block "-%}" such that any trailing whitespace and newline are suppressed + /// + CodeSuppressWsExit, } [StructLayout(LayoutKind.Auto)] internal readonly record struct CharPosition(int Line, int Column, int Offset) { public static readonly CharPosition Empty = new(0, 0, -1); + + public Token ToToken(TokenType type, int count) + { + return new Token( + this, + new CharPosition(Line, Column + count - 1, Offset + count - 1), + type + ); + } } [StructLayout(LayoutKind.Auto)] @@ -52,6 +91,44 @@ public bool IsRawToken() return Type is TokenType.Raw or TokenType.Whitespace or TokenType.Newline; } + public bool IsCodeBlockEnterToken() + { + return Type is TokenType.CodeEnter or TokenType.CodeSuppressWsEnter; + } + + public bool IsCodeBlockExitToken() + { + return Type is TokenType.CodeExit or TokenType.CodeSuppressWsExit; + } + + public bool IsRenderBlockEnterToken() + { + return Type is TokenType.RenderEnter or TokenType.RenderSuppressWsEnter; + } + + public bool IsRenderBlockExitToken() + { + return Type is TokenType.RenderExit or TokenType.RenderSuppressWsExit; + } + + public bool IsBlockEnterToken() + { + return Type + is TokenType.RenderEnter + or TokenType.RenderSuppressWsEnter + or TokenType.CodeEnter + or TokenType.CodeSuppressWsEnter; + } + + public bool IsBlockExitToken() + { + return Type + is TokenType.RenderExit + or TokenType.RenderSuppressWsExit + or TokenType.CodeExit + or TokenType.CodeSuppressWsExit; + } + public override string ToString() { return $"{Type}({Start}:{End})"; diff --git a/Cutout/Parser/Parser.ParseBlock.cs b/Cutout/Parser/Parser.ParseBlock.cs new file mode 100644 index 0000000..ba1e6bf --- /dev/null +++ b/Cutout/Parser/Parser.ParseBlock.cs @@ -0,0 +1,103 @@ +namespace Cutout; + +internal static partial class Parser +{ + private sealed class BlockContext + { + private Context Context { get; set; } + public int StartIndex { get; private set; } + public int ExitIndex { get; set; } + public int Length { get; set; } + public int RawTextCount { get; set; } + public int IdentifierIndex { get; set; } + + public ReadOnlySpan Identifier => + Context.Tokens[IdentifierIndex].ToSpan(Context.Template); + public bool IsJustIdentifier => RawTextCount == 1; + + public BlockContext(Context context) + { + Context = context; + Reset(context); + } + + public bool IsOnlyIdentifier(in ReadOnlySpan match) + { + return IsJustIdentifier && Identifier.SequenceEqual(match); + } + + public bool IsIdentifier(in ReadOnlySpan match) + { + return Identifier.SequenceEqual(match); + } + + public TokenList RemainingTokens() + { + var remainingCount = ExitIndex - 1 - IdentifierIndex; + return Context.Tokens.GetRange(IdentifierIndex + 1, remainingCount); + } + + public void Reset(Context context) + { + Context = context; + StartIndex = context.Index + 1; + ExitIndex = -1; + Length = 0; + IdentifierIndex = -1; + RawTextCount = 0; + } + } + + private static void ParseBlock(Context context, BlockContext blockContext) + { + blockContext.Reset(context); + while (context.MoveNext()) + { + blockContext.RawTextCount += context.Current.Type == TokenType.Raw ? 1 : 0; + + if (context.Current.IsBlockExitToken()) + { + var startBlockToken = context.Tokens[blockContext.StartIndex - 1]; + if ( + startBlockToken.IsCodeBlockEnterToken() + && context.Current.IsRenderBlockExitToken() + ) + { + throw context.Failure("Render block exit token cannot be used in code blocks"); + } + if ( + startBlockToken.IsRenderBlockEnterToken() + && context.Current.IsCodeBlockExitToken() + ) + { + throw context.Failure("Code block exit token cannot be used in render blocks"); + } + + blockContext.ExitIndex = context.Index; + break; + } + + blockContext.Length++; + + if (context.Current.IsBlockEnterToken()) + { + throw context.Failure("Nested code or render blocks are not allowed"); + } + + if (blockContext.IdentifierIndex < 0 && context.Current.Type == TokenType.Raw) + { + blockContext.IdentifierIndex = context.Index; + } + } + + if (blockContext.IdentifierIndex < 0) + { + throw context.Failure(blockContext.StartIndex, "Code block is empty"); + } + + if (blockContext.ExitIndex < 0) + { + throw context.Failure(blockContext.Length - 1, "Code exit token not found"); + } + } +} diff --git a/Cutout/Parser/Parser.ParseCodeBlock.cs b/Cutout/Parser/Parser.ParseCodeBlock.cs deleted file mode 100644 index 752e11c..0000000 --- a/Cutout/Parser/Parser.ParseCodeBlock.cs +++ /dev/null @@ -1,87 +0,0 @@ -namespace Cutout; - -internal static partial class Parser -{ - private sealed class CodeBlockContext - { - private Context Context { get; set; } - public int StartIndex { get; private set; } - public int CodeExitIndex { get; set; } - public int Length { get; set; } - public int RawTextCount { get; set; } - public int IdentifierIndex { get; set; } - - public ReadOnlySpan Identifier => - Context.Tokens[IdentifierIndex].ToSpan(Context.Template); - public bool IsJustIdentifier => RawTextCount == 1; - - public CodeBlockContext(Context context) - { - Context = context; - Reset(context); - } - - public bool IsOnlyIdentifier(in ReadOnlySpan match) - { - return IsJustIdentifier && Identifier.SequenceEqual(match); - } - - public bool IsIdentifier(in ReadOnlySpan match) - { - return Identifier.SequenceEqual(match); - } - - public TokenList RemainingTokens() - { - var remainingCount = CodeExitIndex - 1 - IdentifierIndex; - return Context.Tokens.GetRange(IdentifierIndex + 1, remainingCount); - } - - public void Reset(Context context) - { - Context = context; - StartIndex = context.Index + 1; - CodeExitIndex = -1; - Length = 0; - IdentifierIndex = -1; - RawTextCount = 0; - } - } - - private static void ParseCodeBlock(Context context, CodeBlockContext codeBlockContext) - { - codeBlockContext.Reset(context); - while (context.MoveNext()) - { - codeBlockContext.RawTextCount += context.Current.Type == TokenType.Raw ? 1 : 0; - - if (context.Current.Type == TokenType.CodeExit) - { - codeBlockContext.CodeExitIndex = context.Index; - break; - } - - codeBlockContext.Length++; - - if (context.Current.Type == TokenType.CodeEnter) - { - throw context.Failure("Nested code blocks are not allowed"); - } - - if (codeBlockContext.IdentifierIndex < 0 && context.Current.Type == TokenType.Raw) - { - codeBlockContext.IdentifierIndex = context.Index; - } - } - - if (codeBlockContext.IdentifierIndex < 0) - { - throw context.Failure(codeBlockContext.StartIndex, "Code block is empty"); - } - - if (codeBlockContext.CodeExitIndex < 0) - { - throw context.Failure(codeBlockContext.Length - 1, "Code exit token not found"); - } - } -} diff --git a/Cutout/Parser/Parser.cs b/Cutout/Parser/Parser.cs index 4b9edae..9d2f1de 100644 --- a/Cutout/Parser/Parser.cs +++ b/Cutout/Parser/Parser.cs @@ -42,10 +42,10 @@ private enum BreakOn private static SyntaxList ParseInternal( Context context, BreakOn breakOn, - out CodeBlockContext? endBlockContext + out BlockContext? endBlockContext ) { - CodeBlockContext? blockContext = null; + BlockContext? blockContext = null; var syntaxList = new SyntaxList(); while (context.MoveNext()) @@ -55,14 +55,15 @@ out CodeBlockContext? endBlockContext case TokenType.Eof: if (breakOn != BreakOn.Eof) { - throw context.Failure("Unexpected end of file, expected a {{ end }} block"); + throw context.Failure("Unexpected end of file, expected a {% end %} block"); } continue; case TokenType.CodeEnter: + case TokenType.CodeSuppressWsEnter: { - blockContext ??= new CodeBlockContext(context); - ParseCodeBlock(context, blockContext); + blockContext ??= new BlockContext(context); + ParseBlock(context, blockContext); if (TryParseRecursiveEnd(context, blockContext, breakOn)) { @@ -132,15 +133,27 @@ out _ } else { - var codeTokens = context.Tokens.GetRange( - blockContext.StartIndex, - blockContext.Length + context.Failure( + blockContext.IdentifierIndex, + $"Unknown code block: {{% {blockContext.Identifier.ToString()} %}}" ); - var syntax = new Syntax.RenderableExpression(codeTokens); - syntaxList.Add(syntax); } break; } + case TokenType.RenderEnter: + case TokenType.RenderSuppressWsEnter: + { + blockContext ??= new BlockContext(context); + ParseBlock(context, blockContext); + + var codeTokens = context.Tokens.GetRange( + blockContext.StartIndex, + blockContext.Length + ); + var syntax = new Syntax.RenderableExpression(codeTokens); + syntaxList.Add(syntax); + break; + } default: { var rawText = ParseRawText(context); @@ -156,32 +169,32 @@ out _ private static bool TryParseRecursiveEnd( Context context, - CodeBlockContext codeBlockContext, + BlockContext blockContext, BreakOn breakOn ) { if ( - !codeBlockContext.IsIdentifier(End) - && !codeBlockContext.IsIdentifier(ElseIf) - && !codeBlockContext.IsIdentifier(Else) + !blockContext.IsIdentifier(End) + && !blockContext.IsIdentifier(ElseIf) + && !blockContext.IsIdentifier(Else) ) { return false; } - if (!codeBlockContext.IsJustIdentifier && !codeBlockContext.IsIdentifier(ElseIf)) + if (!blockContext.IsJustIdentifier && !blockContext.IsIdentifier(ElseIf)) { throw context.Failure( - codeBlockContext.IdentifierIndex, - $"{{{{ {codeBlockContext.Identifier.ToString()} }}}} statement should only contain the identifier" + blockContext.IdentifierIndex, + $"{{% {blockContext.Identifier.ToString()} %}} statement should only contain the identifier" ); } if (breakOn == BreakOn.Eof) { throw context.Failure( - codeBlockContext.IdentifierIndex, - $"{{{{ {codeBlockContext.Identifier.ToString()} }}}} found but not expected" + blockContext.IdentifierIndex, + $"{{% {blockContext.Identifier.ToString()} %}} found but not expected" ); } @@ -190,7 +203,7 @@ BreakOn breakOn private static bool TryParseVarStatement( Context context, - CodeBlockContext blockContext, + BlockContext blockContext, out Syntax.VarStatement? syntax ) { @@ -204,7 +217,7 @@ out Syntax.VarStatement? syntax { throw context.Failure( blockContext.IdentifierIndex, - "{{ var }} declaration requires an assignment expression" + "{% var %} declaration requires an assignment expression" ); } @@ -214,7 +227,7 @@ out Syntax.VarStatement? syntax private static bool TryParseCallStatement( Context context, - CodeBlockContext blockContext, + BlockContext blockContext, out Syntax.CallStatement? syntax ) { @@ -228,18 +241,18 @@ out Syntax.CallStatement? syntax { throw context.Failure( blockContext.IdentifierIndex, - "{{ call }} statement requires parameters" + "{% call %} statement requires parameters" ); } - var text = blockContext.RemainingTokens().ToSpan(context.Template).Trim().ToString(); + var text = blockContext.RemainingTokens().ToString(context.Template).Trim(); var callParts = text.Split('(', ')'); if (callParts.Length != 3 || string.IsNullOrWhiteSpace(callParts[0])) { throw context.Failure( blockContext.IdentifierIndex, - "{{ call }} statement requires a function name and () with optional parameters" + "{% call %} statement requires a function name and () with optional parameters" ); } @@ -256,17 +269,17 @@ out Syntax.CallStatement? syntax private static void ParseConditionalStatement( Context context, - CodeBlockContext blockContext, + BlockContext blockContext, out TokenList condition, out SyntaxList expressions, - out CodeBlockContext? endBlockContext + out BlockContext? endBlockContext ) { if (blockContext.IsJustIdentifier) { throw context.Failure( blockContext.IdentifierIndex, - $"{{{{ {blockContext.Identifier.ToString()} }}}} statement requires a condition" + $"{{% {blockContext.Identifier.ToString()} %}} statement requires a condition" ); } @@ -276,7 +289,7 @@ out CodeBlockContext? endBlockContext private static bool TryParseIfStatement( Context context, - CodeBlockContext blockContext, + BlockContext blockContext, out Syntax.IfStatement? syntax ) { @@ -290,7 +303,7 @@ out Syntax.IfStatement? syntax { throw context.Failure( blockContext.IdentifierIndex, - "{{ if }} statement requires a condition" + "{% if %} statement requires a condition" ); } @@ -312,7 +325,7 @@ out var endBlockContext { throw context.Failure( endBlockContext.IdentifierIndex, - "Cannot have {{ elseif }} after {{ else }} in an {{ if }} statement" + "Cannot have {% elseif %} after {% else %} in an {% if %} statement" ); } @@ -334,7 +347,7 @@ out endBlockContext { throw context.Failure( endBlockContext.IdentifierIndex, - "Only one {{ else }} is allowed within an {{ if }} statement" + "Only one {% else %} is allowed within an {% if %} statement" ); } diff --git a/Cutout/Parser/Syntax.cs b/Cutout/Parser/Syntax.cs index ab58371..bae6350 100644 --- a/Cutout/Parser/Syntax.cs +++ b/Cutout/Parser/Syntax.cs @@ -4,7 +4,10 @@ internal abstract record Syntax { private Syntax() { } - internal sealed record RawText(TokenList Value) : Syntax; + internal sealed record RawText(TokenList Value) : Syntax + { + public bool ContainsNewLine => Value.Exists(static x => x.Type == TokenType.Newline); + } internal sealed record RenderableExpression(TokenList Value) : Syntax; diff --git a/Cutout/README.md b/Cutout/README.md index e0d040b..3b7209f 100644 --- a/Cutout/README.md +++ b/Cutout/README.md @@ -37,11 +37,11 @@ using Cutout; public static partial class MyTemplate { private const string Template = """ - {{ if name == "Bob" }} + {% if name == "Bob" %} Hello Bob - {{ else }} + {% else %} Hello {{ name }} - {{ end }} + {% end %} """; [Cutout.Template(Template)] @@ -51,13 +51,16 @@ public static partial class MyTemplate ## Template Language -Everything that is not between `{{` and `}}` is treated as a string literal. -any expression that does not start with a known keyword is treated as a variable -to be rendered. +Everything that is not between `{{` or `{@` and `}}` or `%}` is treated as a +string literal. Any valid C# expression can be used in the template as it is compiled to C# code. So if something is not working, the compiler will tell you. +It supports the whitespace control characters `-` the same as liquid. + +One deviation from liquid is that there is only one end keyword, `{% end %}`. + The following keywords are supported, ### If/Else @@ -68,13 +71,13 @@ Everything that is a valid C# boolean expression can be used. For example, ```liquid -{{ if true }} +{% if true %} Some that is true -{{ else if false }} +{% elseif false %} Some other thing -{{ else }} +{% else %} The default -{{ end }} +{% end %} ``` ### For/Each/While @@ -83,16 +86,16 @@ The standard looping keywords are supported. They align to the same keywords in C#. ```liquid -{{ for i = 0; i < items.Count; i++ }} +{% for i = 0; i < items.Count; i++ %} {{ i }} -{{ end }} +{% end %} -{{ foreach item in items }} +{% foreach item in items %} {{ item }} -{{ end }} +{% end %} -{{ while true }} -{{ end }} +{% while true %} +{% end %} ``` `var`, `continue`, `break` and `return` are also supported. @@ -106,5 +109,5 @@ This allows for building up complex templates from smaller ones. The syntax is like so, ```liquid -{{ call MyFunction(1, 2, 3) }} +{% call MyFunction(1, 2, 3) %} ``` diff --git a/Cutout/Renderer/Renderer.cs b/Cutout/Renderer/Renderer.cs index 976b08c..45d67ec 100644 --- a/Cutout/Renderer/Renderer.cs +++ b/Cutout/Renderer/Renderer.cs @@ -1,6 +1,4 @@ using System.CodeDom.Compiler; -using Cutout.Extensions; -using Cutout.Parser; namespace Cutout; @@ -8,6 +6,7 @@ internal static class Renderer { internal static void WriteSyntax( this IndentedTextWriter writer, + string template, Syntax syntax, bool includeWhitespaceReceiver ) @@ -18,13 +17,19 @@ bool includeWhitespaceReceiver writer.WriteLine($"var {varStatement.Assignment};"); break; case Syntax.CallStatement callStatement: - writer.WriteCallStatement(callStatement, includeWhitespaceReceiver); + writer.WriteCallStatement(template, callStatement, includeWhitespaceReceiver); break; case Syntax.IfStatement ifStatement: - writer.WriteConditionalStatement(ifStatement, "if (", includeWhitespaceReceiver); + writer.WriteConditionalStatement( + template, + ifStatement, + "if (", + includeWhitespaceReceiver + ); break; case Syntax.ElseIfStatement elseIfStatement: writer.WriteConditionalStatement( + template, elseIfStatement, "else if (", includeWhitespaceReceiver @@ -32,10 +37,15 @@ bool includeWhitespaceReceiver break; case Syntax.ElseStatement elseStatement: writer.WriteLine("else"); - writer.WriteExpressions(elseStatement.Expressions, includeWhitespaceReceiver); + writer.WriteExpressions( + template, + elseStatement.Expressions, + includeWhitespaceReceiver + ); break; case Syntax.ForStatement forStatement: writer.WriteConditionalStatement( + template, forStatement, "for (var ", includeWhitespaceReceiver @@ -43,6 +53,7 @@ bool includeWhitespaceReceiver break; case Syntax.ForeachStatement foreachStatement: writer.WriteConditionalStatement( + template, foreachStatement, "foreach (var ", includeWhitespaceReceiver @@ -50,6 +61,7 @@ bool includeWhitespaceReceiver break; case Syntax.WhileStatement whileStatement: writer.WriteConditionalStatement( + template, whileStatement, "while (", includeWhitespaceReceiver @@ -65,25 +77,30 @@ bool includeWhitespaceReceiver writer.WriteLine("return;"); break; case Syntax.RawText rawText: - writer.WriteRawText(rawText, includeWhitespaceReceiver); + writer.WriteRawText(template, rawText, includeWhitespaceReceiver); break; case Syntax.RenderableExpression renderableExpression: - writer.WriteRenderableExpression(renderableExpression, includeWhitespaceReceiver); + writer.WriteRenderableExpression( + template, + renderableExpression, + includeWhitespaceReceiver + ); break; } } private static void WriteRawText( this IndentedTextWriter writer, + string template, Syntax.RawText rawText, bool includeWhitespaceReceiver ) { writer.Write("builder.Append(@\""); - // writer.Write(rawText.Value); + writer.Write(rawText.Value.ToSpan(template).ToString()); writer.WriteLine("\");"); - if (includeWhitespaceReceiver && rawText.ContainsNewline) + if (includeWhitespaceReceiver && rawText.ContainsNewLine) { writer.WriteLine("builder.Append(whitespace);"); } @@ -91,18 +108,19 @@ bool includeWhitespaceReceiver private static void WriteRenderableExpression( this IndentedTextWriter writer, + string template, Syntax.RenderableExpression renderableExpression, bool includeWhitespaceReceiver ) { writer.Write("builder.Append("); - writer.Write(renderableExpression.Value); + writer.Write(renderableExpression.Value.ToSpan(template).ToString()); writer.WriteLine(");"); if (includeWhitespaceReceiver) { writer.Write("if (("); - writer.Write(renderableExpression.Value); + writer.Write(renderableExpression.Value.ToSpan(template).ToString()); writer.WriteLine(").ToString().IndexOf('\\n') != -1)"); writer.WriteLine("{"); using (writer.Indent()) @@ -115,6 +133,7 @@ bool includeWhitespaceReceiver private static void WriteExpressions( this IndentedTextWriter writer, + string template, IReadOnlyList expressions, bool includeWhitespaceReceiver ) @@ -125,7 +144,7 @@ bool includeWhitespaceReceiver for (var i = 0; i < expressions.Count; i++) { var syntax = expressions[i]; - WriteSyntax(writer, syntax, includeWhitespaceReceiver); + WriteSyntax(writer, template, syntax, includeWhitespaceReceiver); } } writer.WriteLine("}"); @@ -133,6 +152,7 @@ bool includeWhitespaceReceiver private static void WriteConditionalStatement( this IndentedTextWriter writer, + string template, Syntax.ConditionalStatement forStatement, string conditionalPrefix, bool includeWhitespaceReceiver @@ -142,11 +162,12 @@ bool includeWhitespaceReceiver writer.Write(forStatement.Condition); writer.WriteLine(")"); - WriteExpressions(writer, forStatement.Expressions, includeWhitespaceReceiver); + WriteExpressions(writer, template, forStatement.Expressions, includeWhitespaceReceiver); } private static void WriteCallStatement( this IndentedTextWriter writer, + string template, Syntax.CallStatement callStatement, bool includeWhitespaceReceiver ) diff --git a/Cutout/TemplateAttributeParts.cs b/Cutout/TemplateAttributeParts.cs index 925cdbb..638ac04 100644 --- a/Cutout/TemplateAttributeParts.cs +++ b/Cutout/TemplateAttributeParts.cs @@ -22,8 +22,7 @@ public TemplateAttributeParts(MethodDetails details, SemanticModel ctxSemanticMo Template = template.HasValue ? template.Value?.ToString() : string.Empty; - var templateSpan = Template.AsSpan(); - var tokens = Lexer.Tokenize(templateSpan); + var tokens = Lexer.Tokenize(Template ?? string.Empty); // Syntaxes = TemplateParser.Parse(tokens, templateSpan); } } diff --git a/README.md b/README.md index ed7312c..dd09a14 100644 --- a/README.md +++ b/README.md @@ -53,11 +53,11 @@ using Cutout; public static partial class MyTemplate { private const string Template = """ - {{ if name == "Bob" }} + {% if name == "Bob" %} Hello Bob - {{ else }} + {% else %} Hello {{ name }} - {{ end }} + {% end %} """; [Cutout.Template(Template)] @@ -67,13 +67,16 @@ public static partial class MyTemplate ## Template Language -Everything that is not between `{{` and `}}` is treated as a string literal. -any expression that does not start with a known keyword is treated as a variable -to be rendered. +Everything that is not between `{{` or `{@` and `}}` or `%}` is treated as a +string literal. Any valid C# expression can be used in the template as it is compiled to C# code. So if something is not working, the compiler will tell you. +It supports the whitespace control characters `-` the same as liquid. + +One deviation from liquid is that there is only one end keyword, `{% end %}`. + The following keywords are supported, ### If/Else @@ -84,13 +87,13 @@ Everything that is a valid C# boolean expression can be used. For example, ```liquid -{{ if true }} +{% if true %} Some that is true -{{ else if false }} +{% elseif false %} Some other thing -{{ else }} +{% else %} The default -{{ end }} +{% end %} ``` ### For/Each/While @@ -99,16 +102,16 @@ The standard looping keywords are supported. They align to the same keywords in C#. ```liquid -{{ for i = 0; i < items.Count; i++ }} +{% for i = 0; i < items.Count; i++ %} {{ i }} -{{ end }} +{% end %} -{{ foreach item in items }} +{% foreach item in items %} {{ item }} -{{ end }} +{% end %} -{{ while true }} -{{ end }} +{% while true %} +{% end %} ``` `var`, `continue`, `break` and `return` are also supported. @@ -122,5 +125,5 @@ This allows for building up complex templates from smaller ones. The syntax is like so, ```liquid -{{ call MyFunction(1, 2, 3) }} +{% call MyFunction(1, 2, 3) %} ``` From 6a53f580149574523913c0c402d0327fa9935e93 Mon Sep 17 00:00:00 2001 From: bmazzarol Date: Sat, 2 Aug 2025 14:07:21 +0800 Subject: [PATCH 08/10] chore: update library versions --- .globalconfig | 2 + Common.Benchmark.props | 42 ++++---- Common.Docs.props | 86 ++++++++--------- Common.Release.props | 66 ++++++------- Common.Test.props | 66 ++++++------- Cutout.Sample/Cutout.Sample.csproj | 22 +++-- Cutout.Tests/Cutout.Tests.csproj | 114 +++++++++++----------- Cutout/Cutout.csproj | 89 +++++++++-------- Directory.Packages.props | 7 +- Parent.Directory.Packages.props | 150 +++++++++++++++-------------- 10 files changed, 318 insertions(+), 326 deletions(-) diff --git a/.globalconfig b/.globalconfig index 3cc5598..051af15 100644 --- a/.globalconfig +++ b/.globalconfig @@ -20,3 +20,5 @@ dotnet_diagnostic.S1135.severity = suggestion # logging performance dotnet_diagnostic.CA1848.severity = none +# Add at least one assertion to this test case. +dotnet_diagnostic.S2699.severity = suggestion \ No newline at end of file diff --git a/Common.Benchmark.props b/Common.Benchmark.props index f096057..ef5fcc4 100644 --- a/Common.Benchmark.props +++ b/Common.Benchmark.props @@ -1,28 +1,24 @@  - - - - net8.0 - true - $(AssemblyName.Replace('.Benchmarks', '')) - Exe - false - - - - - - - - - - - - - - - + + net8.0 + true + $(AssemblyName.Replace('.Benchmarks', '')) + Exe + false + + + + + + + + + + + + diff --git a/Common.Docs.props b/Common.Docs.props index c3fa351..cef4d5b 100644 --- a/Common.Docs.props +++ b/Common.Docs.props @@ -1,51 +1,49 @@  - - + + net8.0 + + + - net8.0 + false - - - - - false - - - - - true - - - - - - - - - - - - - - - - - - - - + + + true + + + + + + + + + + + + + + diff --git a/Common.Release.props b/Common.Release.props index 8bc8b5b..cb328b1 100644 --- a/Common.Release.props +++ b/Common.Release.props @@ -1,40 +1,34 @@  - - - - Ben Mazzarol - true - MIT - Copyright (c) Ben Mazzarol. All rights reserved. - - - - true - snupkg - true - True - bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml - README.md - latest - library - - - - - - - - - - - - - - <_Parameter1>$(AssemblyName).Tests - - - - \ No newline at end of file + + Ben Mazzarol + true + MIT + Copyright (c) Ben Mazzarol. All rights reserved. + + + true + snupkg + true + True + bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml + README.md + latest + library + + + + + + + + + + + <_Parameter1>$(AssemblyName).Tests + + + diff --git a/Common.Test.props b/Common.Test.props index e88d7a2..65a3c38 100644 --- a/Common.Test.props +++ b/Common.Test.props @@ -1,40 +1,36 @@  - - - - net8.0 - true - $(AssemblyName.Replace('.Tests', '')) - Exe - true - true - - - - - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - - - - - - - - - - + + net8.0 + true + $(AssemblyName.Replace('.Tests', '')) + Exe + true + true + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + diff --git a/Cutout.Sample/Cutout.Sample.csproj b/Cutout.Sample/Cutout.Sample.csproj index 039afdb..05417b5 100644 --- a/Cutout.Sample/Cutout.Sample.csproj +++ b/Cutout.Sample/Cutout.Sample.csproj @@ -1,12 +1,14 @@  - - - net8.0 - enable - Cutout.Sample - - - - - + + net8.0 + enable + Cutout.Sample + + + + diff --git a/Cutout.Tests/Cutout.Tests.csproj b/Cutout.Tests/Cutout.Tests.csproj index 0e073ee..51139c0 100644 --- a/Cutout.Tests/Cutout.Tests.csproj +++ b/Cutout.Tests/Cutout.Tests.csproj @@ -1,61 +1,57 @@ - - - - - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Cutout/Cutout.csproj b/Cutout/Cutout.csproj index 87fe16a..69d057d 100644 --- a/Cutout/Cutout.csproj +++ b/Cutout/Cutout.csproj @@ -1,48 +1,45 @@ - - - - - netstandard2.0 - Cutout - Zero cost compile time templating - C#, StringBuilder, Templating, Liquid - https://bmazzarol.github.io/Cutout - https://github.com/bmazzarol/Cutout - scissors-icon.png - true - true - true - true - false - RS1035;MA0037 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + netstandard2.0 + Cutout + Zero cost compile time templating + C#, StringBuilder, Templating, Liquid + https://bmazzarol.github.io/Cutout + https://github.com/bmazzarol/Cutout + scissors-icon.png + true + true + true + true + false + RS1035;MA0037 + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Directory.Packages.props b/Directory.Packages.props index b894b88..0cf5163 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -2,6 +2,9 @@ $([MSBuild]::GetPathOfFileAbove('Directory.Packages.props', '$(MSBuildThisFileDirectory)/../')) - + - \ No newline at end of file + diff --git a/Parent.Directory.Packages.props b/Parent.Directory.Packages.props index 3f1e3c1..7418038 100644 --- a/Parent.Directory.Packages.props +++ b/Parent.Directory.Packages.props @@ -1,73 +1,81 @@ - - false - true - true - enable - enable - Recommended - true - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers - - - all - runtime; build; native; contentfiles; analyzers - - - all - runtime; build; native; contentfiles; analyzers - - - all - runtime; build; native; contentfiles; analyzers - - + + false + true + true + enable + enable + Recommended + true + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + all + runtime; build; native; contentfiles; analyzers + + + all + runtime; build; native; contentfiles; analyzers + + + all + runtime; build; native; contentfiles; analyzers + + From 42d85f50516922202d3f0e323a2468a439d1c92c Mon Sep 17 00:00:00 2001 From: bmazzarol Date: Sat, 2 Aug 2025 22:09:23 +0800 Subject: [PATCH 09/10] fix: enable all tests and ensure it all works --- ...tementTests.Case1a#Test.Test.g.verified.cs | 2 +- ...ntTests.Case1a#Test.Test.wsr.g.verified.cs | 2 +- Cutout.Tests/CallStatementTests.cs | 6 +- Cutout.Tests/Cutout.Tests.csproj | 35 -- ...tementTests.Case1a#Test.Test.g.verified.cs | 2 +- ...ntTests.Case1a#Test.Test.wsr.g.verified.cs | 2 +- ...tementTests.Case2a#Test.Test.g.verified.cs | 4 +- ...ntTests.Case2a#Test.Test.wsr.g.verified.cs | 6 +- ...tementTests.Case3a#Test.Test.g.verified.cs | 8 +- ...ntTests.Case3a#Test.Test.wsr.g.verified.cs | 10 +- Cutout.Tests/ForStatementTests.cs | 57 +- ...tementTests.Case1a#Test.Test.g.verified.cs | 2 +- ...ntTests.Case1a#Test.Test.wsr.g.verified.cs | 2 +- ...tementTests.Case2a#Test.Test.g.verified.cs | 2 +- ...ntTests.Case2a#Test.Test.wsr.g.verified.cs | 2 +- ...tementTests.Case3a#Test.Test.g.verified.cs | 4 +- ...ntTests.Case3a#Test.Test.wsr.g.verified.cs | 4 +- Cutout.Tests/IfStatementTests.cs | 24 +- Cutout.Tests/ParserTests.cs | 42 +- Cutout.Tests/TemplateParserTests.cs | 492 ------------------ ...ase2a#Test.TestWithParameter.g.verified.cs | 2 +- ...a#Test.TestWithParameter.wsr.g.verified.cs | 4 +- ...a#Test.TestWithTwoParameters.g.verified.cs | 4 +- ...st.TestWithTwoParameters.wsr.g.verified.cs | 8 +- ...ts.Case4a#Test.TestWithModel.g.verified.cs | 4 +- ...ase4a#Test.TestWithModel.wsr.g.verified.cs | 8 +- ...est.TestWithConstantTemplate.g.verified.cs | 2 +- ...TestWithConstantTemplate.wsr.g.verified.cs | 2 +- Cutout.Tests/TemplateTests.cs | 12 +- Cutout/Cutout.csproj | 7 - Cutout/Exceptions/ParseException.cs | 6 +- Cutout/Parser/Parser.Context.cs | 12 +- Cutout/Parser/Parser.cs | 6 +- Cutout/Parser/Syntax.cs | 24 + Cutout/Renderer/Renderer.cs | 80 +-- Cutout/TemplateAttributeParts.cs | 3 +- ...plateSourceGenerator.TemplateMethodImpl.cs | 10 +- Cutout/TemplateSourceGenerator.cs | 4 + 38 files changed, 212 insertions(+), 694 deletions(-) delete mode 100644 Cutout.Tests/TemplateParserTests.cs diff --git a/Cutout.Tests/CallStatementTests.Case1a#Test.Test.g.verified.cs b/Cutout.Tests/CallStatementTests.Case1a#Test.Test.g.verified.cs index 9f3af6c..6b4d739 100644 --- a/Cutout.Tests/CallStatementTests.Case1a#Test.Test.g.verified.cs +++ b/Cutout.Tests/CallStatementTests.Case1a#Test.Test.g.verified.cs @@ -13,6 +13,6 @@ internal static partial class Test internal static partial void Test(this StringBuilder builder, String product) { builder.Append(@"Some text before "); - Case2(builder,product); + Case2(builder, product); } } diff --git a/Cutout.Tests/CallStatementTests.Case1a#Test.Test.wsr.g.verified.cs b/Cutout.Tests/CallStatementTests.Case1a#Test.Test.wsr.g.verified.cs index 781c2e0..7ec8b56 100644 --- a/Cutout.Tests/CallStatementTests.Case1a#Test.Test.wsr.g.verified.cs +++ b/Cutout.Tests/CallStatementTests.Case1a#Test.Test.wsr.g.verified.cs @@ -13,6 +13,6 @@ internal static partial class Test internal static void Test(this StringBuilder builder, String product, string whitespace) { builder.Append(@"Some text before "); - Case2(builder,product); + Case2(builder, product); } } diff --git a/Cutout.Tests/CallStatementTests.cs b/Cutout.Tests/CallStatementTests.cs index 1b6efb3..4eefcf4 100644 --- a/Cutout.Tests/CallStatementTests.cs +++ b/Cutout.Tests/CallStatementTests.cs @@ -7,7 +7,7 @@ public static partial class CallTemplates { public sealed record Product(string Title); - private const string CallExample1 = "Some text before {{ call Case2(product.Title) }}"; + private const string CallExample1 = "Some text before {% call Case2(product.Title) %}"; [Template(CallExample1)] public static partial void Case1(this StringBuilder builder, Product product); @@ -20,7 +20,7 @@ public sealed record Product(string Title); private const string CallExample3 = """ This is an example with a call with leading whitespace, ``` - {{ call Case4(product) }} + {% call Case4(product) %} ``` """; @@ -50,7 +50,7 @@ public void Case1() [Fact(DisplayName = "Case1 produces the expected source")] public Task Case1a() => """ - [Template("Some text before {{ call Case2(product) }}")] + [Template("Some text before {% call Case2(product) %}")] public static partial void Test(this StringBuilder builder, string product); """.VerifyTemplate(); diff --git a/Cutout.Tests/Cutout.Tests.csproj b/Cutout.Tests/Cutout.Tests.csproj index 51139c0..29f8aa9 100644 --- a/Cutout.Tests/Cutout.Tests.csproj +++ b/Cutout.Tests/Cutout.Tests.csproj @@ -19,39 +19,4 @@ ReferenceOutputAssembly="true" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Cutout.Tests/ForStatementTests.Case1a#Test.Test.g.verified.cs b/Cutout.Tests/ForStatementTests.Case1a#Test.Test.g.verified.cs index b595aa1..02a2cc6 100644 --- a/Cutout.Tests/ForStatementTests.Case1a#Test.Test.g.verified.cs +++ b/Cutout.Tests/ForStatementTests.Case1a#Test.Test.g.verified.cs @@ -13,7 +13,7 @@ internal static partial class Test internal static partial void Test(this StringBuilder builder, String product) { builder.Append(@"This is a test for tags ["); - foreach (var tag in product.Tags) + foreach (var tag in product.Tags ) { builder.Append(tag); builder.Append(@"; "); diff --git a/Cutout.Tests/ForStatementTests.Case1a#Test.Test.wsr.g.verified.cs b/Cutout.Tests/ForStatementTests.Case1a#Test.Test.wsr.g.verified.cs index 77b09aa..24c2d2f 100644 --- a/Cutout.Tests/ForStatementTests.Case1a#Test.Test.wsr.g.verified.cs +++ b/Cutout.Tests/ForStatementTests.Case1a#Test.Test.wsr.g.verified.cs @@ -13,7 +13,7 @@ internal static partial class Test internal static void Test(this StringBuilder builder, String product, string whitespace) { builder.Append(@"This is a test for tags ["); - foreach (var tag in product.Tags) + foreach (var tag in product.Tags ) { builder.Append(tag); if ((tag).ToString().IndexOf('\n') != -1) diff --git a/Cutout.Tests/ForStatementTests.Case2a#Test.Test.g.verified.cs b/Cutout.Tests/ForStatementTests.Case2a#Test.Test.g.verified.cs index d808680..6ad556f 100644 --- a/Cutout.Tests/ForStatementTests.Case2a#Test.Test.g.verified.cs +++ b/Cutout.Tests/ForStatementTests.Case2a#Test.Test.g.verified.cs @@ -13,11 +13,11 @@ internal static partial class Test internal static partial void Test(this StringBuilder builder, String product) { builder.Append(@"This is a test for tags ["); - while (i < product.Tags.Length) + while ( i < product.Tags.Length ) { builder.Append(i + 1); builder.Append(@". "); - builder.Append(product.Tags[i++]); + builder.Append( product.Tags[i++] ); } builder.Append(@"] which is cool."); } diff --git a/Cutout.Tests/ForStatementTests.Case2a#Test.Test.wsr.g.verified.cs b/Cutout.Tests/ForStatementTests.Case2a#Test.Test.wsr.g.verified.cs index 978daac..d341e93 100644 --- a/Cutout.Tests/ForStatementTests.Case2a#Test.Test.wsr.g.verified.cs +++ b/Cutout.Tests/ForStatementTests.Case2a#Test.Test.wsr.g.verified.cs @@ -13,7 +13,7 @@ internal static partial class Test internal static void Test(this StringBuilder builder, String product, string whitespace) { builder.Append(@"This is a test for tags ["); - while (i < product.Tags.Length) + while ( i < product.Tags.Length ) { builder.Append(i + 1); if ((i + 1).ToString().IndexOf('\n') != -1) @@ -21,8 +21,8 @@ internal static void Test(this StringBuilder builder, String product, string whi builder.Append(whitespace); } builder.Append(@". "); - builder.Append(product.Tags[i++]); - if ((product.Tags[i++]).ToString().IndexOf('\n') != -1) + builder.Append( product.Tags[i++] ); + if (( product.Tags[i++] ).ToString().IndexOf('\n') != -1) { builder.Append(whitespace); } diff --git a/Cutout.Tests/ForStatementTests.Case3a#Test.Test.g.verified.cs b/Cutout.Tests/ForStatementTests.Case3a#Test.Test.g.verified.cs index 43c5100..91addee 100644 --- a/Cutout.Tests/ForStatementTests.Case3a#Test.Test.g.verified.cs +++ b/Cutout.Tests/ForStatementTests.Case3a#Test.Test.g.verified.cs @@ -13,19 +13,19 @@ internal static partial class Test internal static partial void Test(this StringBuilder builder, String product) { builder.Append(@"This is a test for tags ["); - for (var i = 0; i < product.Tags.Length; i++) + for (var i = 0; i < product.Tags.Length; i++ ) { - if (product.Tags[i] == "awesome") + if ( product.Tags[i] == "awesome" ) { continue; } - else if (product.Tags[i] == "shoes") + else if ( product.Tags[i] == "shoes" ) { break; } builder.Append(i + 1); builder.Append(@". "); - builder.Append(product.Tags[i]); + builder.Append( product.Tags[i] ); } builder.Append(@"] which is cool."); } diff --git a/Cutout.Tests/ForStatementTests.Case3a#Test.Test.wsr.g.verified.cs b/Cutout.Tests/ForStatementTests.Case3a#Test.Test.wsr.g.verified.cs index a5e70cf..2ff7022 100644 --- a/Cutout.Tests/ForStatementTests.Case3a#Test.Test.wsr.g.verified.cs +++ b/Cutout.Tests/ForStatementTests.Case3a#Test.Test.wsr.g.verified.cs @@ -13,13 +13,13 @@ internal static partial class Test internal static void Test(this StringBuilder builder, String product, string whitespace) { builder.Append(@"This is a test for tags ["); - for (var i = 0; i < product.Tags.Length; i++) + for (var i = 0; i < product.Tags.Length; i++ ) { - if (product.Tags[i] == "awesome") + if ( product.Tags[i] == "awesome" ) { continue; } - else if (product.Tags[i] == "shoes") + else if ( product.Tags[i] == "shoes" ) { break; } @@ -29,8 +29,8 @@ internal static void Test(this StringBuilder builder, String product, string whi builder.Append(whitespace); } builder.Append(@". "); - builder.Append(product.Tags[i]); - if ((product.Tags[i]).ToString().IndexOf('\n') != -1) + builder.Append( product.Tags[i] ); + if (( product.Tags[i] ).ToString().IndexOf('\n') != -1) { builder.Append(whitespace); } diff --git a/Cutout.Tests/ForStatementTests.cs b/Cutout.Tests/ForStatementTests.cs index 835d4da..b347f52 100644 --- a/Cutout.Tests/ForStatementTests.cs +++ b/Cutout.Tests/ForStatementTests.cs @@ -14,19 +14,21 @@ public sealed record Product(string Title, string[] Tags); These are the tags, - {{ foreach tag in product.Tags }} + {%- foreach tag in product.Tags %} * {{ tag }} - {{ end }} + {%- end %} + These can also be numbered, - {{ foreach (tag, index) in product.Tags.Select((tag,index) => (tag, index + 1)) }} - {{index}}. {{ tag }} - {{ end }} + {%- foreach (tag, index) in product.Tags.Select((tag,index) => (tag, index + 1)) %} + {{ index }}. {{ tag }} + {%- end %} + And used with traditional for loop, - {{ for i = 0; i < product.Tags.Length; i++ }} - {{i + 1}}. {{ product.Tags[i] }} - {{ end }} + {%- for i = 0; i < product.Tags.Length; i++ %} + {{ i + 1 }}. {{ product.Tags[i] }} + {%- end %} """; [Template(ForExample1)] @@ -35,10 +37,10 @@ public sealed record Product(string Title, string[] Tags); private const string ForExample2 = """ A while loop can also be used - {{ var i = 0 }} - {{ while i < product.Tags.Length }} - {{i + 1}}. {{ product.Tags[i++] }} - {{ end }} + {% var i = 0 -%} + {% while i < product.Tags.Length -%} + {{ i + 1 }}. {{ product.Tags[i++] }} + {% end %} """; [Template(ForExample2)] @@ -46,15 +48,15 @@ public sealed record Product(string Title, string[] Tags); private const string ForExample3 = """ Continue and break can also be used - {{ for i = 0; i < product.Tags.Length; i++ }} - {{ if product.Tags[i] == "awesome" }} - {{ continue }} - {{ end }} - {{ if product.Tags[i] == "shoes" }} - {{ break }} - {{ end }} + {%- for i = 0; i < product.Tags.Length; i++ -%} + {%- if product.Tags[i] == "awesome" -%} + {%- continue -%} + {%- end -%} + {%- if product.Tags[i] == "shoes" -%} + {%- break -%} + {%- end -%} {{i + 1}}. {{ product.Tags[i] }} - {{ end }} + {%- end -%} """; [Template(ForExample3)] @@ -91,7 +93,6 @@ 3. cool 1. awesome 2. shoes 3. cool - """, builder.ToString() ); @@ -100,7 +101,7 @@ 3. cool [Fact(DisplayName = "Case1 produces the expected source")] public Task Case1a() => """ - [Template("This is a test for tags [{{ foreach tag in product.Tags }}{{tag}}; {{ end }}] which is cool.")] + [Template("This is a test for tags [{% foreach tag in product.Tags %}{{tag}}; {% end %}] which is cool.")] public static partial void Test(this StringBuilder builder, string product); """.VerifyTemplate(); @@ -125,7 +126,7 @@ 3. cool [Fact(DisplayName = "Case2 produces the expected source")] public Task Case2a() => """ - [Template("This is a test for tags [{{ while i < product.Tags.Length }}{{i + 1}}. {{ product.Tags[i++] }}{{ end }}] which is cool.")] + [Template("This is a test for tags [{% while i < product.Tags.Length %}{{i + 1}}. {{ product.Tags[i++] }}{% end %}] which is cool.")] public static partial void Test(this StringBuilder builder, string product); """.VerifyTemplate(); @@ -134,19 +135,13 @@ public void Case3() { var builder = new StringBuilder(); builder.Case3(new ForTemplates.Product("Awesome Shoes", ["awesome", "shoes", "cool"])); - Assert.Equal( - """ - Continue and break can also be used - - """, - builder.ToString() - ); + Assert.Equal("Continue and break can also be used", builder.ToString()); } [Fact(DisplayName = "Case3 produces the expected source")] public Task Case3a() => """ - [Template("This is a test for tags [{{ for i = 0; i < product.Tags.Length; i++ }}{{ if product.Tags[i] == \"awesome\" }}{{ continue }}{{ else if product.Tags[i] == \"shoes\" }}{{ break }}{{ end }}{{i + 1}}. {{ product.Tags[i] }}{{ end }}] which is cool.")] + [Template("This is a test for tags [{% for i = 0; i < product.Tags.Length; i++ %}{% if product.Tags[i] == \"awesome\" %}{% continue %}{% elseif product.Tags[i] == \"shoes\" %}{% break %}{% end %}{{i + 1}}. {{ product.Tags[i] }}{% end %}] which is cool.")] public static partial void Test(this StringBuilder builder, string product); """.VerifyTemplate(); } diff --git a/Cutout.Tests/IfStatementTests.Case1a#Test.Test.g.verified.cs b/Cutout.Tests/IfStatementTests.Case1a#Test.Test.g.verified.cs index 76d0032..8e2be18 100644 --- a/Cutout.Tests/IfStatementTests.Case1a#Test.Test.g.verified.cs +++ b/Cutout.Tests/IfStatementTests.Case1a#Test.Test.g.verified.cs @@ -12,7 +12,7 @@ internal static partial class Test { internal static partial void Test(this StringBuilder builder, String product) { - if (product == "Awesome Shoes") + if ( product == "Awesome Shoes" ) { builder.Append(@" These shoes are awesome! "); } diff --git a/Cutout.Tests/IfStatementTests.Case1a#Test.Test.wsr.g.verified.cs b/Cutout.Tests/IfStatementTests.Case1a#Test.Test.wsr.g.verified.cs index c3e918e..eaf3c58 100644 --- a/Cutout.Tests/IfStatementTests.Case1a#Test.Test.wsr.g.verified.cs +++ b/Cutout.Tests/IfStatementTests.Case1a#Test.Test.wsr.g.verified.cs @@ -12,7 +12,7 @@ internal static partial class Test { internal static void Test(this StringBuilder builder, String product, string whitespace) { - if (product == "Awesome Shoes") + if ( product == "Awesome Shoes" ) { builder.Append(@" These shoes are awesome! "); } diff --git a/Cutout.Tests/IfStatementTests.Case2a#Test.Test.g.verified.cs b/Cutout.Tests/IfStatementTests.Case2a#Test.Test.g.verified.cs index 2c17d3d..870f4ff 100644 --- a/Cutout.Tests/IfStatementTests.Case2a#Test.Test.g.verified.cs +++ b/Cutout.Tests/IfStatementTests.Case2a#Test.Test.g.verified.cs @@ -12,7 +12,7 @@ internal static partial class Test { internal static partial void Test(this StringBuilder builder, String product) { - if (product == "Awesome Shoes") + if ( product == "Awesome Shoes" ) { builder.Append(@" These shoes are awesome! "); } diff --git a/Cutout.Tests/IfStatementTests.Case2a#Test.Test.wsr.g.verified.cs b/Cutout.Tests/IfStatementTests.Case2a#Test.Test.wsr.g.verified.cs index 8fe33cb..b95860a 100644 --- a/Cutout.Tests/IfStatementTests.Case2a#Test.Test.wsr.g.verified.cs +++ b/Cutout.Tests/IfStatementTests.Case2a#Test.Test.wsr.g.verified.cs @@ -12,7 +12,7 @@ internal static partial class Test { internal static void Test(this StringBuilder builder, String product, string whitespace) { - if (product == "Awesome Shoes") + if ( product == "Awesome Shoes" ) { builder.Append(@" These shoes are awesome! "); } diff --git a/Cutout.Tests/IfStatementTests.Case3a#Test.Test.g.verified.cs b/Cutout.Tests/IfStatementTests.Case3a#Test.Test.g.verified.cs index 95a87f7..e2e2362 100644 --- a/Cutout.Tests/IfStatementTests.Case3a#Test.Test.g.verified.cs +++ b/Cutout.Tests/IfStatementTests.Case3a#Test.Test.g.verified.cs @@ -12,11 +12,11 @@ internal static partial class Test { internal static partial void Test(this StringBuilder builder, String product) { - if (product == "Awesome Shoes") + if ( product == "Awesome Shoes" ) { builder.Append(@" These shoes are awesome! "); } - else if (product == "Cool Shoes") + else if ( product == "Cool Shoes" ) { builder.Append(@" These shoes are cool! "); } diff --git a/Cutout.Tests/IfStatementTests.Case3a#Test.Test.wsr.g.verified.cs b/Cutout.Tests/IfStatementTests.Case3a#Test.Test.wsr.g.verified.cs index 8f27c5d..d245c8e 100644 --- a/Cutout.Tests/IfStatementTests.Case3a#Test.Test.wsr.g.verified.cs +++ b/Cutout.Tests/IfStatementTests.Case3a#Test.Test.wsr.g.verified.cs @@ -12,11 +12,11 @@ internal static partial class Test { internal static void Test(this StringBuilder builder, String product, string whitespace) { - if (product == "Awesome Shoes") + if ( product == "Awesome Shoes" ) { builder.Append(@" These shoes are awesome! "); } - else if (product == "Cool Shoes") + else if ( product == "Cool Shoes" ) { builder.Append(@" These shoes are cool! "); } diff --git a/Cutout.Tests/IfStatementTests.cs b/Cutout.Tests/IfStatementTests.cs index fc1b901..3b91b9b 100644 --- a/Cutout.Tests/IfStatementTests.cs +++ b/Cutout.Tests/IfStatementTests.cs @@ -8,33 +8,33 @@ public static partial class IfTemplates public sealed record Product(string Title); private const string IfExample1 = """ - {{ if product.Title == "Awesome Shoes" }} + {% if product.Title == "Awesome Shoes" -%} These shoes are awesome! - {{ end }} + {%- end -%} """; [Template(IfExample1)] public static partial void Case1(this StringBuilder builder, Product product); private const string IfExample2 = """ - {{ if product.Title == "Awesome Shoes" }} + {% if product.Title == "Awesome Shoes" -%} These shoes are awesome! - {{ else }} + {%- else -%} These shoes are not awesome! - {{ end }} + {%- end -%} """; [Template(IfExample2)] public static partial void Case2(this StringBuilder builder, Product product); private const string IfExample3 = """ - {{ if product.Title == "Awesome Shoes" }} + {% if product.Title == "Awesome Shoes" -%} These shoes are awesome! - {{ else if product.Title == "Cool Shoes" }} + {%- elseif product.Title == "Cool Shoes" -%} These shoes are cool! - {{ else }} + {%- else -%} These shoes are not awesome or cool! - {{ end }} + {%- end -%} """; [Template(IfExample3)] @@ -54,7 +54,7 @@ public void Case1() [Fact(DisplayName = "Case1 produces the expected source")] public Task Case1a() => """ - [Template("{{ if product == \"Awesome Shoes\" }} These shoes are awesome! {{ end }}")] + [Template("{% if product == \"Awesome Shoes\" %} These shoes are awesome! {% end %}")] public static partial void Test(this StringBuilder builder, string product); """.VerifyTemplate(); @@ -73,7 +73,7 @@ public void Case2() [Fact(DisplayName = "Case2 produces the expected source")] public Task Case2a() => """ - [Template("{{ if product == \"Awesome Shoes\" }} These shoes are awesome! {{ else }} These shoes are not awesome! {{ end }}")] + [Template("{% if product == \"Awesome Shoes\" %} These shoes are awesome! {% else %} These shoes are not awesome! {% end %}")] public static partial void Test(this StringBuilder builder, string product); """.VerifyTemplate(); @@ -96,7 +96,7 @@ public void Case3() [Fact(DisplayName = "Case3 produces the expected source")] public Task Case3a() => """ - [Template("{{ if product == \"Awesome Shoes\" }} These shoes are awesome! {{ else if product == \"Cool Shoes\" }} These shoes are cool! {{ else }} These shoes are not awesome or cool! {{ end }}")] + [Template("{% if product == \"Awesome Shoes\" %} These shoes are awesome! {% elseif product == \"Cool Shoes\" %} These shoes are cool! {% else %} These shoes are not awesome or cool! {% end %}")] public static partial void Test(this StringBuilder builder, string product); """.VerifyTemplate(); } diff --git a/Cutout.Tests/ParserTests.cs b/Cutout.Tests/ParserTests.cs index 7af13ea..9d90a98 100644 --- a/Cutout.Tests/ParserTests.cs +++ b/Cutout.Tests/ParserTests.cs @@ -18,17 +18,17 @@ public void Case1() token => { Assert.Equal(TokenType.Raw, token.Type); - Assert.Equal(token.ToSpan(template), "raw"); + Assert.Equal("raw", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); + Assert.Equal(" ", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Raw, token.Type); - Assert.Equal(token.ToSpan(template), "string"); + Assert.Equal("string", token.ToSpan(template)); } ); } @@ -47,17 +47,17 @@ public void Case2() token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); + Assert.Equal(" ", token.ToSpan(template)); }, token => { + Assert.Equal("code", token.ToSpan(template)); Assert.Equal(TokenType.Raw, token.Type); - Assert.Equal(token.ToSpan(template), "code"); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); + Assert.Equal(" ", token.ToSpan(template)); } ); } @@ -246,37 +246,37 @@ public void Case11() token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); + Assert.Equal(" ", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Raw, token.Type); - Assert.Equal(token.ToSpan(template), "x"); + Assert.Equal("x", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); + Assert.Equal(" ", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Raw, token.Type); - Assert.Equal(token.ToSpan(template), "="); + Assert.Equal("=", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); + Assert.Equal(" ", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Raw, token.Type); - Assert.Equal(token.ToSpan(template), "42"); + Assert.Equal("42", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); + Assert.Equal(" ", token.ToSpan(template)); } ); } @@ -415,17 +415,17 @@ public void Case17() token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); + Assert.Equal(" ", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Raw, token.Type); - Assert.Equal(token.ToSpan(template), "condition"); + Assert.Equal("condition", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); + Assert.Equal(" ", token.ToSpan(template)); } ); @@ -436,27 +436,27 @@ public void Case17() token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); + Assert.Equal(" ", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Raw, token.Type); - Assert.Equal(token.ToSpan(template), "some"); + Assert.Equal("some", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); + Assert.Equal(" ", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Raw, token.Type); - Assert.Equal(token.ToSpan(template), "code"); + Assert.Equal("code", token.ToSpan(template)); }, token => { Assert.Equal(TokenType.Whitespace, token.Type); - Assert.Equal(token.ToSpan(template), " "); + Assert.Equal(" ", token.ToSpan(template)); } ); } diff --git a/Cutout.Tests/TemplateParserTests.cs b/Cutout.Tests/TemplateParserTests.cs deleted file mode 100644 index 05f5fa7..0000000 --- a/Cutout.Tests/TemplateParserTests.cs +++ /dev/null @@ -1,492 +0,0 @@ -using Cutout.Exceptions; -using Cutout.Parser; - -namespace Cutout.Tests; - -public sealed class TemplateParserTests -{ - [Fact(DisplayName = "A raw string can be parsed")] - public void Case1() - { - const string template = "raw string"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - var item = Assert.Single(result); - var rawText = Assert.IsType(item); - Assert.Equal("raw string", rawText.Value); - } - - [Fact(DisplayName = "A renderable code block can be parsed")] - public void Case2() - { - const string template = "{{ code }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - var item = Assert.Single(result); - var renderableExpression = Assert.IsType(item); - Assert.Equal("code", renderableExpression.Value); - } - - [Fact(DisplayName = "Raw and renderable code blocks can be parsed")] - public void Case3() - { - const string template = "raw {{ code }} string"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - Assert.Collection( - result, - item => - { - var rawText = Assert.IsType(item); - Assert.Equal("raw ", rawText.Value); - }, - item => - { - var renderableExpression = Assert.IsType(item); - Assert.Equal("code", renderableExpression.Value); - }, - item => - { - var rawText = Assert.IsType(item); - Assert.Equal(" string", rawText.Value); - } - ); - } - - [Fact(DisplayName = "An if statement without else can be parsed")] - public void Case4() - { - const string template = "{{ if condition }} test {{ end }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - var item = Assert.Single(result); - var ifStatement = Assert.IsType(item); - Assert.Equal("condition", ifStatement.Condition); - var expression = Assert.Single(ifStatement.Expressions); - var rawText = Assert.IsType(expression); - Assert.Equal(" test ", rawText.Value); - } - - [Fact(DisplayName = "An if statement with else can be parsed")] - public void Case5() - { - const string template = "{{ if condition }} test {{ else }} test2 {{ end }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - Assert.Collection( - result, - item => - { - var ifStatement = Assert.IsType(item); - Assert.Equal("condition", ifStatement.Condition); - var expression = Assert.Single(ifStatement.Expressions); - var rawText = Assert.IsType(expression); - Assert.Equal(" test ", rawText.Value); - }, - item => - { - var elseStatement = Assert.IsType(item); - var expression = Assert.Single(elseStatement.Expressions); - var rawText = Assert.IsType(expression); - Assert.Equal(" test2 ", rawText.Value); - } - ); - } - - [Fact(DisplayName = "An if statement with multiple conditions can be parsed")] - public void Case6() - { - const string template = - "{{ if condition1 }} test {{ else if condition2 }} test2 {{ else }} test3 {{ end }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - Assert.Collection( - result, - item => - { - var ifStatement = Assert.IsType(item); - Assert.Equal("condition1", ifStatement.Condition); - var expression = Assert.Single(ifStatement.Expressions); - var rawText = Assert.IsType(expression); - Assert.Equal(" test ", rawText.Value); - }, - item => - { - var elseIfStatement = Assert.IsType(item); - Assert.Equal("condition2", elseIfStatement.Condition); - var expression = Assert.Single(elseIfStatement.Expressions); - var rawText = Assert.IsType(expression); - Assert.Equal(" test2 ", rawText.Value); - }, - item => - { - var elseStatement = Assert.IsType(item); - var expression = Assert.Single(elseStatement.Expressions); - var rawText = Assert.IsType(expression); - Assert.Equal(" test3 ", rawText.Value); - } - ); - } - - [Fact(DisplayName = "An if statement with multiple conditions and no else can be parsed")] - public void Case7() - { - const string template = "{{ if condition1 }} test {{ else if condition2 }} test2 {{ end }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - Assert.Collection( - result, - item => - { - var ifStatement = Assert.IsType(item); - Assert.Equal("condition1", ifStatement.Condition); - var expression = Assert.Single(ifStatement.Expressions); - var rawText = Assert.IsType(expression); - Assert.Equal(" test ", rawText.Value); - }, - item => - { - var elseIfStatement = Assert.IsType(item); - Assert.Equal("condition2", elseIfStatement.Condition); - var expression = Assert.Single(elseIfStatement.Expressions); - var rawText = Assert.IsType(expression); - Assert.Equal(" test2 ", rawText.Value); - } - ); - } - - [Fact(DisplayName = "An invalid if statement will throw an exception (else too short)")] - public void Case8() - { - const string template = "{{ if condition1 }} test {{ else if }} test2 {{ end }}"; - var tokens = Lexer.Tokenize(template); - var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); - Assert.Equal( - "Parse error at 0:36 (CodeExit): else if statement condition not found (value: '}}')", - exception.Message - ); - } - - [Fact(DisplayName = "An invalid if statement will throw an exception (else missing if)")] - public void Case9() - { - const string template = - "{{ if condition1 }} test {{ else invalid condition }} test2 {{ end }}"; - var tokens = Lexer.Tokenize(template); - var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); - Assert.Equal( - "Parse error at 0:33 (Identifier): else statement must be followed by if statement (value: 'invalid')", - exception.Message - ); - } - - [Fact(DisplayName = "An invalid if statement will throw an exception (if too short)")] - public void Case10() - { - const string template = "{{ if }} test {{ end }}"; - var tokens = Lexer.Tokenize(template); - var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); - Assert.Equal( - "Parse error at 0:6 (CodeExit): if statement condition not found (value: '}}')", - exception.Message - ); - } - - [Fact(DisplayName = "An invalid if statement will throw an exception (end missing)")] - public void Case11() - { - const string template = "{{ if condition1 }} test {{ else if condition2 }} test2 "; - var tokens = Lexer.Tokenize(template); - var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); - Assert.Equal( - "Parse error at 0:49 (Raw): else or end not found (value: ' test2 ')", - exception.Message - ); - } - - [Fact(DisplayName = "An if statement can be nested")] - public void Case12() - { - const string template = "{{ if condition1 }}{{ if condition2 }} test {{ end }}{{ end }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - var item = Assert.Single(result); - var ifStatement = Assert.IsType(item); - Assert.Equal("condition1", ifStatement.Condition); - var innerIfStatement = Assert.IsType(ifStatement.Expressions[0]); - Assert.Equal("condition2", innerIfStatement.Condition); - var expression = Assert.Single(innerIfStatement.Expressions); - var rawText = Assert.IsType(expression); - Assert.Equal(" test ", rawText.Value); - } - - [Fact(DisplayName = "A for statement can be parsed")] - public void Case13() - { - const string template = "{{ for condition }} test {{ end }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - var item = Assert.Single(result); - var forStatement = Assert.IsType(item); - Assert.Equal("condition", forStatement.Condition); - var expression = Assert.Single(forStatement.Expressions); - var rawText = Assert.IsType(expression); - Assert.Equal(" test ", rawText.Value); - } - - [Fact(DisplayName = "A invalid for statement will throw an exception (for missing condition)")] - public void Case14() - { - const string template = "{{ for }} test {{ end }}"; - var tokens = Lexer.Tokenize(template); - var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); - Assert.Equal( - "Parse error at 0:7 (CodeExit): for statement condition not found (value: '}}')", - exception.Message - ); - } - - [Fact(DisplayName = "A foreach statement can be parsed")] - public void Case15() - { - const string template = "{{ foreach condition }} test {{ end }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - var item = Assert.Single(result); - var foreachStatement = Assert.IsType(item); - Assert.Equal("condition", foreachStatement.Condition); - var expression = Assert.Single(foreachStatement.Expressions); - var rawText = Assert.IsType(expression); - Assert.Equal(" test ", rawText.Value); - } - - [Fact( - DisplayName = "A invalid foreach statement will throw an exception (foreach missing condition)" - )] - public void Case16() - { - const string template = "{{ foreach }} test {{ end }}"; - var tokens = Lexer.Tokenize(template); - var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); - Assert.Equal( - "Parse error at 0:11 (CodeExit): foreach statement condition not found (value: '}}')", - exception.Message - ); - } - - [Fact(DisplayName = "A while statement can be parsed")] - public void Case17() - { - const string template = "{{ while condition }} test {{ end }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - var item = Assert.Single(result); - var whileStatement = Assert.IsType(item); - Assert.Equal("condition", whileStatement.Condition); - var expression = Assert.Single(whileStatement.Expressions); - var rawText = Assert.IsType(expression); - Assert.Equal(" test ", rawText.Value); - } - - [Fact( - DisplayName = "A invalid while statement will throw an exception (while missing condition)" - )] - public void Case18() - { - const string template = "{{ while }} test {{ end }}"; - var tokens = Lexer.Tokenize(template); - var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); - Assert.Equal( - "Parse error at 0:9 (CodeExit): while statement condition not found (value: '}}')", - exception.Message - ); - } - - [Fact(DisplayName = "A continue statement can be parsed")] - public void Case19() - { - const string template = "{{ continue }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - var item = Assert.Single(result); - Assert.IsType(item); - } - - [Fact(DisplayName = "A break statement can be parsed")] - public void Case20() - { - const string template = "{{ break }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - var item = Assert.Single(result); - Assert.IsType(item); - } - - [Fact(DisplayName = "A invalid break will throw an exception (invalid token)")] - public void Case21() - { - const string template = "{{ break invalid }}"; - var tokens = Lexer.Tokenize(template); - var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); - Assert.Equal( - "Parse error at 0:9 (Identifier): Expected only keyword 'break' (value: 'invalid')", - exception.Message - ); - } - - [Fact(DisplayName = "A while with continue and break can be parsed")] - public void Case22() - { - const string template = """ - {{ while condition }} - {{ if i == 0 }} - {{ continue }} - {{ end }} - - The value is {{i}} - {{ end }} - """; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - var whileStatement = Assert.IsType(Assert.Single(result)); - Assert.Equal("condition", whileStatement.Condition); - Assert.Collection( - whileStatement.Expressions, - item => - { - var ifStatement = Assert.IsType(item); - Assert.Equal("i == 0", ifStatement.Condition); - var expression = Assert.Single(ifStatement.Expressions); - Assert.IsType(expression); - }, - item => - { - var rawText = Assert.IsType(item); - Assert.Equal("\r\n\r\nThe value is ", rawText.Value); - }, - item => - { - var renderableExpression = Assert.IsType(item); - Assert.Equal("i", renderableExpression.Value); - }, - item => - { - var rawText = Assert.IsType(item); - Assert.Equal("\r\n", rawText.Value); - } - ); - } - - [Fact(DisplayName = "A return statement can be parsed")] - public void Case23() - { - const string template = "{{ return }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - var item = Assert.Single(result); - Assert.IsType(item); - } - - [Fact(DisplayName = "A invalid return will throw an exception (invalid token)")] - public void Case24() - { - const string template = "{{ return invalid }}"; - var tokens = Lexer.Tokenize(template); - var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); - Assert.Equal( - "Parse error at 0:10 (Identifier): Expected only keyword 'return' (value: 'invalid')", - exception.Message - ); - } - - [Fact(DisplayName = "Extra newline raw text is removed after code blocks")] - public void Case25() - { - const string template = """ - {{var a = 1}} - {{break}} - {{continue}} - {{return}} - """; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - Assert.Collection( - result, - item => - { - var renderableExpression = Assert.IsType(item); - Assert.Equal("a = 1", renderableExpression.Assignment); - }, - item => Assert.IsType(item), - item => Assert.IsType(item), - item => Assert.IsType(item) - ); - } - - [Fact(DisplayName = "A call statement can be parsed")] - public void Case26() - { - const string template = "{{ call method(param1, param2) }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - var item = Assert.Single(result); - var callStatement = Assert.IsType(item); - Assert.Equal("method", callStatement.Name); - Assert.Equal("param1, param2", callStatement.Parameters); - Assert.Equal(string.Empty, callStatement.LeadingWhitespace); - } - - [Fact(DisplayName = "A invalid call statement will throw an exception (invalid token)")] - public void Case27() - { - const string template = "{{ call method(param1, param2) invalid }}"; - var tokens = Lexer.Tokenize(template); - var exception = Assert.Throws(() => TemplateParser.Parse(tokens, template)); - Assert.Equal( - "Parse error at 0:31 (Identifier): Invalid call statement. Expected format: 'MethodName(...)'. (value: 'invalid')", - exception.Message - ); - } - - [Fact(DisplayName = "A call statement with leading whitespace can be parsed")] - public void Case28() - { - const string template = - "some text \n some other text \n\n {{ call method(param1, param2) }}"; - var tokens = Lexer.Tokenize(template); - var result = TemplateParser.Parse(tokens, template); - - Assert.Collection( - result, - item => - { - var rawText = Assert.IsType(item); - Assert.Equal("some text \n some other text \n\n ", rawText.Value); - }, - item => - { - var callStatement = Assert.IsType(item); - Assert.Equal("method", callStatement.Name); - Assert.Equal("param1, param2", callStatement.Parameters); - Assert.Equal(" ", callStatement.LeadingWhitespace); - } - ); - } -} diff --git a/Cutout.Tests/TemplateTests.Case2a#Test.TestWithParameter.g.verified.cs b/Cutout.Tests/TemplateTests.Case2a#Test.TestWithParameter.g.verified.cs index 60d2aff..f0c9e94 100644 --- a/Cutout.Tests/TemplateTests.Case2a#Test.TestWithParameter.g.verified.cs +++ b/Cutout.Tests/TemplateTests.Case2a#Test.TestWithParameter.g.verified.cs @@ -13,6 +13,6 @@ internal static partial class Test internal static partial void TestWithParameter(this StringBuilder builder, Int32 param) { builder.Append(@"This is a more complex example with a parameter: "); - builder.Append(param); + builder.Append( param ); } } diff --git a/Cutout.Tests/TemplateTests.Case2a#Test.TestWithParameter.wsr.g.verified.cs b/Cutout.Tests/TemplateTests.Case2a#Test.TestWithParameter.wsr.g.verified.cs index 06b8d9d..427dc84 100644 --- a/Cutout.Tests/TemplateTests.Case2a#Test.TestWithParameter.wsr.g.verified.cs +++ b/Cutout.Tests/TemplateTests.Case2a#Test.TestWithParameter.wsr.g.verified.cs @@ -13,8 +13,8 @@ internal static partial class Test internal static void TestWithParameter(this StringBuilder builder, Int32 param, string whitespace) { builder.Append(@"This is a more complex example with a parameter: "); - builder.Append(param); - if ((param).ToString().IndexOf('\n') != -1) + builder.Append( param ); + if (( param ).ToString().IndexOf('\n') != -1) { builder.Append(whitespace); } diff --git a/Cutout.Tests/TemplateTests.Case3a#Test.TestWithTwoParameters.g.verified.cs b/Cutout.Tests/TemplateTests.Case3a#Test.TestWithTwoParameters.g.verified.cs index d7d775e..001ee0e 100644 --- a/Cutout.Tests/TemplateTests.Case3a#Test.TestWithTwoParameters.g.verified.cs +++ b/Cutout.Tests/TemplateTests.Case3a#Test.TestWithTwoParameters.g.verified.cs @@ -13,8 +13,8 @@ internal static partial class Test internal static partial void TestWithTwoParameters(this StringBuilder builder, Int32 param, String param2) { builder.Append(@"This is a more complex example with a parameter "); - builder.Append(param); + builder.Append( param ); builder.Append(@" and a second parameter "); - builder.Append(param2); + builder.Append( param2 ); } } diff --git a/Cutout.Tests/TemplateTests.Case3a#Test.TestWithTwoParameters.wsr.g.verified.cs b/Cutout.Tests/TemplateTests.Case3a#Test.TestWithTwoParameters.wsr.g.verified.cs index d403214..8010a40 100644 --- a/Cutout.Tests/TemplateTests.Case3a#Test.TestWithTwoParameters.wsr.g.verified.cs +++ b/Cutout.Tests/TemplateTests.Case3a#Test.TestWithTwoParameters.wsr.g.verified.cs @@ -13,14 +13,14 @@ internal static partial class Test internal static void TestWithTwoParameters(this StringBuilder builder, Int32 param, String param2, string whitespace) { builder.Append(@"This is a more complex example with a parameter "); - builder.Append(param); - if ((param).ToString().IndexOf('\n') != -1) + builder.Append( param ); + if (( param ).ToString().IndexOf('\n') != -1) { builder.Append(whitespace); } builder.Append(@" and a second parameter "); - builder.Append(param2); - if ((param2).ToString().IndexOf('\n') != -1) + builder.Append( param2 ); + if (( param2 ).ToString().IndexOf('\n') != -1) { builder.Append(whitespace); } diff --git a/Cutout.Tests/TemplateTests.Case4a#Test.TestWithModel.g.verified.cs b/Cutout.Tests/TemplateTests.Case4a#Test.TestWithModel.g.verified.cs index 34f2069..2a885e1 100644 --- a/Cutout.Tests/TemplateTests.Case4a#Test.TestWithModel.g.verified.cs +++ b/Cutout.Tests/TemplateTests.Case4a#Test.TestWithModel.g.verified.cs @@ -13,8 +13,8 @@ internal static partial class Test internal static partial void TestWithModel(this StringBuilder builder, SomeModel model) { builder.Append(@"This is a more complex example with a parameter "); - builder.Append(model.Value); + builder.Append( model.Value ); builder.Append(@" and a second parameter "); - builder.Append(model.Text); + builder.Append( model.Text ); } } diff --git a/Cutout.Tests/TemplateTests.Case4a#Test.TestWithModel.wsr.g.verified.cs b/Cutout.Tests/TemplateTests.Case4a#Test.TestWithModel.wsr.g.verified.cs index 46de661..bd84b32 100644 --- a/Cutout.Tests/TemplateTests.Case4a#Test.TestWithModel.wsr.g.verified.cs +++ b/Cutout.Tests/TemplateTests.Case4a#Test.TestWithModel.wsr.g.verified.cs @@ -13,14 +13,14 @@ internal static partial class Test internal static void TestWithModel(this StringBuilder builder, SomeModel model, string whitespace) { builder.Append(@"This is a more complex example with a parameter "); - builder.Append(model.Value); - if ((model.Value).ToString().IndexOf('\n') != -1) + builder.Append( model.Value ); + if (( model.Value ).ToString().IndexOf('\n') != -1) { builder.Append(whitespace); } builder.Append(@" and a second parameter "); - builder.Append(model.Text); - if ((model.Text).ToString().IndexOf('\n') != -1) + builder.Append( model.Text ); + if (( model.Text ).ToString().IndexOf('\n') != -1) { builder.Append(whitespace); } diff --git a/Cutout.Tests/TemplateTests.Case5a#Test.TestWithConstantTemplate.g.verified.cs b/Cutout.Tests/TemplateTests.Case5a#Test.TestWithConstantTemplate.g.verified.cs index 98a333b..f545735 100644 --- a/Cutout.Tests/TemplateTests.Case5a#Test.TestWithConstantTemplate.g.verified.cs +++ b/Cutout.Tests/TemplateTests.Case5a#Test.TestWithConstantTemplate.g.verified.cs @@ -15,7 +15,7 @@ internal static partial void TestWithConstantTemplate(this StringBuilder builder builder.Append(@"This is an example of a template that is defined in a constant string. "); - if (model.Value > 0) + if ( model.Value > 0 ) { builder.Append(@"The result is positive."); } diff --git a/Cutout.Tests/TemplateTests.Case5a#Test.TestWithConstantTemplate.wsr.g.verified.cs b/Cutout.Tests/TemplateTests.Case5a#Test.TestWithConstantTemplate.wsr.g.verified.cs index 97aa63a..aab696b 100644 --- a/Cutout.Tests/TemplateTests.Case5a#Test.TestWithConstantTemplate.wsr.g.verified.cs +++ b/Cutout.Tests/TemplateTests.Case5a#Test.TestWithConstantTemplate.wsr.g.verified.cs @@ -16,7 +16,7 @@ internal static void TestWithConstantTemplate(this StringBuilder builder, SomeMo "); builder.Append(whitespace); - if (model.Value > 0) + if ( model.Value > 0 ) { builder.Append(@"The result is positive."); } diff --git a/Cutout.Tests/TemplateTests.cs b/Cutout.Tests/TemplateTests.cs index e35d479..182f55a 100644 --- a/Cutout.Tests/TemplateTests.cs +++ b/Cutout.Tests/TemplateTests.cs @@ -32,11 +32,11 @@ This is an example of a template that is defined in a constant string. It has a conditional block, - {{ if model.Value > 0 }} + {% if model.Value > 0 -%} The result is positive. - {{ else }} + {%- else -%} The result is negative. - {{ end }} + {%- end -%} """; [Template(ExampleTemplate)] @@ -146,11 +146,11 @@ public Task Case5a() => """ private const string ExampleTemplate = @"This is an example of a template that is defined in a constant string. - {{ if model.Value > 0 }} + {% if model.Value > 0 -%} The result is positive. - {{ else }} + {%- else -%} The result is negative. - {{ end }} + {%- end -%} "; [Template(ExampleTemplate)] diff --git a/Cutout/Cutout.csproj b/Cutout/Cutout.csproj index 69d057d..dd06542 100644 --- a/Cutout/Cutout.csproj +++ b/Cutout/Cutout.csproj @@ -35,11 +35,4 @@ - - - - - - - diff --git a/Cutout/Exceptions/ParseException.cs b/Cutout/Exceptions/ParseException.cs index 8cfc359..15ae2e0 100644 --- a/Cutout/Exceptions/ParseException.cs +++ b/Cutout/Exceptions/ParseException.cs @@ -1,5 +1,9 @@ -namespace Cutout.Exceptions; +using System.Diagnostics.CodeAnalysis; +namespace Cutout.Exceptions; + +[SuppressMessage("Roslynator", "RCS1194:Implement exception constructors")] +[SuppressMessage("Critical Code Smell", "S3871:Exception types should be \"public\"")] internal sealed class ParseException : Exception { public Token Token { get; } diff --git a/Cutout/Parser/Parser.Context.cs b/Cutout/Parser/Parser.Context.cs index 4ef9d87..4d6c8a0 100644 --- a/Cutout/Parser/Parser.Context.cs +++ b/Cutout/Parser/Parser.Context.cs @@ -30,12 +30,6 @@ public bool MoveNext() return true; } - [ExcludeFromCodeCoverage] - public void Reset() - { - Index = -1; - } - [ExcludeFromCodeCoverage] object IEnumerator.Current => Current; @@ -51,6 +45,12 @@ public ParseException Failure(int index, string reason) return Tokens[index].Failure(Template, reason); } + [ExcludeFromCodeCoverage] + public void Reset() + { + Index = -1; + } + [ExcludeFromCodeCoverage] public void Dispose() { diff --git a/Cutout/Parser/Parser.cs b/Cutout/Parser/Parser.cs index 9d2f1de..b3065db 100644 --- a/Cutout/Parser/Parser.cs +++ b/Cutout/Parser/Parser.cs @@ -85,11 +85,11 @@ out BlockContext? endBlockContext } else if (TryParseVarStatement(context, blockContext, out var varSyntax)) { - syntaxList.Add(varSyntax); + syntaxList.Add(varSyntax!); } else if (TryParseCallStatement(context, blockContext, out var callSyntax)) { - syntaxList.Add(callSyntax); + syntaxList.Add(callSyntax!); } else if (blockContext.IsIdentifier(While)) { @@ -129,7 +129,7 @@ out _ } else if (TryParseIfStatement(context, blockContext, out var ifSyntax)) { - syntaxList.Add(ifSyntax); + syntaxList.Add(ifSyntax!); } else { diff --git a/Cutout/Parser/Syntax.cs b/Cutout/Parser/Syntax.cs index bae6350..bd4023c 100644 --- a/Cutout/Parser/Syntax.cs +++ b/Cutout/Parser/Syntax.cs @@ -7,6 +7,30 @@ private Syntax() { } internal sealed record RawText(TokenList Value) : Syntax { public bool ContainsNewLine => Value.Exists(static x => x.Type == TokenType.Newline); + + /// + /// Returns the leading whitespace if it exists and the previous token is a newline + /// + /// leading whitespace token if it exists + /// >true if leading whitespace exists, otherwise false + public bool TryGetLeadingWhitespace(out Token? leadingWhitespace) + { + leadingWhitespace = null; + + var count = Value.Count; + if (count == 0 || Value[count - 1].Type != TokenType.Whitespace) + { + return false; + } + + if (count <= 1 || Value[count - 2].Type != TokenType.Newline) + { + return false; + } + + leadingWhitespace = Value[count - 1]; + return true; + } } internal sealed record RenderableExpression(TokenList Value) : Syntax; diff --git a/Cutout/Renderer/Renderer.cs b/Cutout/Renderer/Renderer.cs index 45d67ec..bd8486f 100644 --- a/Cutout/Renderer/Renderer.cs +++ b/Cutout/Renderer/Renderer.cs @@ -8,16 +8,23 @@ internal static void WriteSyntax( this IndentedTextWriter writer, string template, Syntax syntax, + Syntax? lastSyntax, bool includeWhitespaceReceiver ) { switch (syntax) { case Syntax.VarStatement varStatement: - writer.WriteLine($"var {varStatement.Assignment};"); + writer.WriteLine($"var {varStatement.Assignment.ToString(template)};"); break; case Syntax.CallStatement callStatement: - writer.WriteCallStatement(template, callStatement, includeWhitespaceReceiver); + writer.WriteCallStatement( + lastSyntax is Syntax.RawText rt && rt.TryGetLeadingWhitespace(out var wst) + ? wst?.ToSpan(template).ToString() + : null, + callStatement, + includeWhitespaceReceiver + ); break; case Syntax.IfStatement ifStatement: writer.WriteConditionalStatement( @@ -26,22 +33,27 @@ bool includeWhitespaceReceiver "if (", includeWhitespaceReceiver ); - break; - case Syntax.ElseIfStatement elseIfStatement: - writer.WriteConditionalStatement( - template, - elseIfStatement, - "else if (", - includeWhitespaceReceiver - ); - break; - case Syntax.ElseStatement elseStatement: - writer.WriteLine("else"); - writer.WriteExpressions( - template, - elseStatement.Expressions, - includeWhitespaceReceiver - ); + + foreach (var ifStatementElseIf in ifStatement.ElseIfs ?? []) + { + writer.WriteConditionalStatement( + template, + ifStatementElseIf, + "else if (", + includeWhitespaceReceiver + ); + } + + if (ifStatement.Else != null) + { + writer.WriteLine("else"); + writer.WriteExpressions( + template, + ifStatement.Else.Expressions, + includeWhitespaceReceiver + ); + } + break; case Syntax.ForStatement forStatement: writer.WriteConditionalStatement( @@ -97,7 +109,7 @@ bool includeWhitespaceReceiver ) { writer.Write("builder.Append(@\""); - writer.Write(rawText.Value.ToSpan(template).ToString()); + writer.Write(rawText.Value.ToString(template)); writer.WriteLine("\");"); if (includeWhitespaceReceiver && rawText.ContainsNewLine) @@ -114,13 +126,13 @@ bool includeWhitespaceReceiver ) { writer.Write("builder.Append("); - writer.Write(renderableExpression.Value.ToSpan(template).ToString()); + writer.Write(renderableExpression.Value.ToString(template)); writer.WriteLine(");"); if (includeWhitespaceReceiver) { writer.Write("if (("); - writer.Write(renderableExpression.Value.ToSpan(template).ToString()); + writer.Write(renderableExpression.Value.ToString(template)); writer.WriteLine(").ToString().IndexOf('\\n') != -1)"); writer.WriteLine("{"); using (writer.Indent()) @@ -144,7 +156,13 @@ bool includeWhitespaceReceiver for (var i = 0; i < expressions.Count; i++) { var syntax = expressions[i]; - WriteSyntax(writer, template, syntax, includeWhitespaceReceiver); + WriteSyntax( + writer, + template, + syntax, + lastSyntax: i < expressions.Count - 1 ? expressions[i + 1] : null, + includeWhitespaceReceiver + ); } } writer.WriteLine("}"); @@ -159,7 +177,7 @@ bool includeWhitespaceReceiver ) { writer.Write(conditionalPrefix); - writer.Write(forStatement.Condition); + writer.Write(forStatement.Condition.ToString(template)); writer.WriteLine(")"); WriteExpressions(writer, template, forStatement.Expressions, includeWhitespaceReceiver); @@ -167,18 +185,22 @@ bool includeWhitespaceReceiver private static void WriteCallStatement( this IndentedTextWriter writer, - string template, + string? whitespace, Syntax.CallStatement callStatement, bool includeWhitespaceReceiver ) { writer.Write(callStatement.Name); - writer.Write("(builder,"); - writer.Write(callStatement.Parameters); - if (!string.IsNullOrEmpty(callStatement.LeadingWhitespace)) + writer.Write("(builder"); + foreach (var parameter in callStatement.Parameters) + { + writer.Write(", "); + writer.Write(parameter); + } + if (!string.IsNullOrEmpty(whitespace)) { - writer.Write(includeWhitespaceReceiver ? ",whitespace + \"" : ",\""); - writer.Write(callStatement.LeadingWhitespace); + writer.Write(includeWhitespaceReceiver ? ", whitespace + \"" : ",\""); + writer.Write(whitespace); writer.Write("\""); } writer.WriteLine(");"); diff --git a/Cutout/TemplateAttributeParts.cs b/Cutout/TemplateAttributeParts.cs index 638ac04..dba10f0 100644 --- a/Cutout/TemplateAttributeParts.cs +++ b/Cutout/TemplateAttributeParts.cs @@ -23,6 +23,7 @@ public TemplateAttributeParts(MethodDetails details, SemanticModel ctxSemanticMo Template = template.HasValue ? template.Value?.ToString() : string.Empty; var tokens = Lexer.Tokenize(Template ?? string.Empty); - // Syntaxes = TemplateParser.Parse(tokens, templateSpan); + var tokensWithWsSuppressed = Lexer.ApplyWhitespaceSuppression(tokens); + Syntaxes = Parser.Parse(tokensWithWsSuppressed, Template ?? string.Empty); } } diff --git a/Cutout/TemplateSourceGenerator.TemplateMethodImpl.cs b/Cutout/TemplateSourceGenerator.TemplateMethodImpl.cs index e86e1eb..33437d3 100644 --- a/Cutout/TemplateSourceGenerator.TemplateMethodImpl.cs +++ b/Cutout/TemplateSourceGenerator.TemplateMethodImpl.cs @@ -93,10 +93,12 @@ bool includeWhitespaceReceiver return; } - // foreach (var syntax in model.AttributeDetails.Syntaxes) - // { - // writer.WriteSyntax(syntax, includeWhitespaceReceiver); - // } + Syntax? lastSyntax = null; + foreach (var syntax in model.AttributeDetails.Syntaxes) + { + writer.WriteSyntax(template, syntax, lastSyntax, includeWhitespaceReceiver); + lastSyntax = syntax; + } } private static void WriteNamespaceParts(IndentedTextWriter writer, TemplateMethodDetails model) diff --git a/Cutout/TemplateSourceGenerator.cs b/Cutout/TemplateSourceGenerator.cs index 0516a58..2859366 100644 --- a/Cutout/TemplateSourceGenerator.cs +++ b/Cutout/TemplateSourceGenerator.cs @@ -8,9 +8,13 @@ namespace Cutout; +/// +/// Source generator for Cutout templates +/// [Generator] public sealed partial class TemplateSourceGenerator : IIncrementalGenerator { + /// public void Initialize(IncrementalGeneratorInitializationContext context) { context.RegisterPostInitializationOutput(ctx => From 8144e29d4d9931e107e49054885588141441fd1b Mon Sep 17 00:00:00 2001 From: bmazzarol Date: Sat, 2 Aug 2025 23:04:17 +0800 Subject: [PATCH 10/10] chore: review fixes 1 --- .globalconfig | 3 + Cutout.Tests/LexerTests.cs | 6 +- Cutout.Tests/ParserTests.cs | 58 +++++++++++++++ Cutout/Cutout.csproj | 5 +- Cutout/Extensions/TokenListExtensions.cs | 8 +++ Cutout/Parser/Parser.Context.cs | 8 +-- Cutout/Parser/Parser.ParseBlock.cs | 29 +++++++- Cutout/Parser/Parser.cs | 91 +++++++++++++++++++++--- Cutout/Parser/Syntax.cs | 3 +- Cutout/TemplateSourceGenerator.cs | 2 + 10 files changed, 187 insertions(+), 26 deletions(-) diff --git a/.globalconfig b/.globalconfig index 051af15..4e3b1cd 100644 --- a/.globalconfig +++ b/.globalconfig @@ -11,6 +11,9 @@ dotnet_diagnostic.MA0004.severity = none # Fix TODO comment dotnet_diagnostic.MA0026.severity = suggestion +# Method is too long +dotnet_diagnostic.MA0051.severity = suggestion + # Remove the unused internal class dotnet_diagnostic.S1144.severity = suggestion diff --git a/Cutout.Tests/LexerTests.cs b/Cutout.Tests/LexerTests.cs index 3bf276b..563236e 100644 --- a/Cutout.Tests/LexerTests.cs +++ b/Cutout.Tests/LexerTests.cs @@ -412,10 +412,10 @@ public void Case5() var tokens = Lexer.Tokenize(text); Assert.Collection( tokens, - tokens => + token => { - Assert.Equal(TokenType.RenderSuppressWsEnter, tokens.Type); - Assert.Equal("{{-", tokens.ToSpan(text).ToString()); + Assert.Equal(TokenType.RenderSuppressWsEnter, token.Type); + Assert.Equal("{{-", token.ToSpan(text).ToString()); }, token => { diff --git a/Cutout.Tests/ParserTests.cs b/Cutout.Tests/ParserTests.cs index 9d90a98..2355287 100644 --- a/Cutout.Tests/ParserTests.cs +++ b/Cutout.Tests/ParserTests.cs @@ -812,4 +812,62 @@ public void Case39() Assert.NotNull(elseNestedIf.Else); Assert.Single(elseNestedIf.Else.Expressions); } + + [Fact(DisplayName = "Unmatched opening delimiter '{{-' throws error")] + public void Case40() + { + const string template = "{{- code"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:5 (Raw): Code exit token not found (value: 'code')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("code", exception.Value); + } + + [Fact(DisplayName = "Unmatched opening delimiter '{%-' throws error")] + public void Case41() + { + const string template = "{%- code"; + var tokens = Lexer.Tokenize(template); + var exception = Assert.Throws(() => Parser.Parse(tokens, template)); + Assert.Equal( + "Parse error at 1:5 (Raw): Code exit token not found (value: 'code')", + exception.Message + ); + Assert.Equal(TokenType.Raw, exception.Token.Type); + Assert.Equal("code", exception.Value); + } + + [Fact(DisplayName = "A call statement with comma in string parameter parses correctly")] + public void Case42() + { + const string template = "{% call function(\"a,b\", 42) %}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + var item = Assert.Single(result); + var callStatement = Assert.IsType(item); + Assert.Equal("function", callStatement.Name); + Assert.Equal(2, callStatement.Parameters.Count); + Assert.Equal("\"a,b\"", callStatement.Parameters[0]); + Assert.Equal("42", callStatement.Parameters[1]); + } + + [Fact( + DisplayName = "A call statement with quoted comma and nested parentheses parses correctly" + )] + public void Case43() + { + const string template = "{% call func(\"value, with, commas\", (1, 2, 3)) %}"; + var tokens = Lexer.Tokenize(template); + var result = Parser.Parse(tokens, template); + var item = Assert.Single(result); + var callStatement = Assert.IsType(item); + Assert.Equal("func", callStatement.Name); + Assert.Equal(2, callStatement.Parameters.Count); + Assert.Equal("\"value, with, commas\"", callStatement.Parameters[0]); + Assert.Equal("(1, 2, 3)", callStatement.Parameters[1]); + } } diff --git a/Cutout/Cutout.csproj b/Cutout/Cutout.csproj index dd06542..e815ad3 100644 --- a/Cutout/Cutout.csproj +++ b/Cutout/Cutout.csproj @@ -11,8 +11,7 @@ true true true - true - false + true RS1035;MA0037 @@ -30,7 +29,7 @@ - + diff --git a/Cutout/Extensions/TokenListExtensions.cs b/Cutout/Extensions/TokenListExtensions.cs index 903214a..7da9b05 100644 --- a/Cutout/Extensions/TokenListExtensions.cs +++ b/Cutout/Extensions/TokenListExtensions.cs @@ -6,6 +6,14 @@ internal static class TokenListExtensions { internal static string ToString(this TokenList list, string template) { + if (string.IsNullOrEmpty(template)) + { + throw new ArgumentException( + "Template string must not be null or empty.", + nameof(template) + ); + } + if (list.Count == 0) { return string.Empty; diff --git a/Cutout/Parser/Parser.Context.cs b/Cutout/Parser/Parser.Context.cs index 4d6c8a0..df1a57c 100644 --- a/Cutout/Parser/Parser.Context.cs +++ b/Cutout/Parser/Parser.Context.cs @@ -46,15 +46,15 @@ public ParseException Failure(int index, string reason) } [ExcludeFromCodeCoverage] - public void Reset() + public void Dispose() { - Index = -1; + Reset(); } [ExcludeFromCodeCoverage] - public void Dispose() + public void Reset() { - Reset(); + Index = -1; } public void Reset(TokenList tokens, string template) diff --git a/Cutout/Parser/Parser.ParseBlock.cs b/Cutout/Parser/Parser.ParseBlock.cs index ba1e6bf..a4be4e4 100644 --- a/Cutout/Parser/Parser.ParseBlock.cs +++ b/Cutout/Parser/Parser.ParseBlock.cs @@ -11,8 +11,17 @@ private sealed class BlockContext public int RawTextCount { get; set; } public int IdentifierIndex { get; set; } - public ReadOnlySpan Identifier => - Context.Tokens[IdentifierIndex].ToSpan(Context.Template); + public ReadOnlySpan Identifier + { + get + { + if (IdentifierIndex < 0 || IdentifierIndex >= Context.Tokens.Count) + { + return []; + } + return Context.Tokens[IdentifierIndex].ToSpan(Context.Template); + } + } public bool IsJustIdentifier => RawTextCount == 1; public BlockContext(Context context) @@ -33,8 +42,22 @@ public bool IsIdentifier(in ReadOnlySpan match) public TokenList RemainingTokens() { + var tokenCount = Context.Tokens.Count; + if ( + IdentifierIndex < 0 + || ExitIndex < 0 + || IdentifierIndex + 1 >= ExitIndex + || IdentifierIndex >= tokenCount + || ExitIndex > tokenCount + ) + { + return []; + } var remainingCount = ExitIndex - 1 - IdentifierIndex; - return Context.Tokens.GetRange(IdentifierIndex + 1, remainingCount); + + return remainingCount <= 0 + ? [] + : Context.Tokens.GetRange(IdentifierIndex + 1, remainingCount); } public void Reset(Context context) diff --git a/Cutout/Parser/Parser.cs b/Cutout/Parser/Parser.cs index b3065db..b2ae03b 100644 --- a/Cutout/Parser/Parser.cs +++ b/Cutout/Parser/Parser.cs @@ -1,4 +1,6 @@ -namespace Cutout; +using System.Text; + +namespace Cutout; internal static partial class Parser { @@ -246,9 +248,10 @@ out Syntax.CallStatement? syntax } var text = blockContext.RemainingTokens().ToString(context.Template).Trim(); - var callParts = text.Split('(', ')'); - if (callParts.Length != 3 || string.IsNullOrWhiteSpace(callParts[0])) + var openParen = text.IndexOf('('); + var closeParen = text.LastIndexOf(')'); + if (openParen < 0 || closeParen < 0 || closeParen < openParen) { throw context.Failure( blockContext.IdentifierIndex, @@ -256,17 +259,83 @@ out Syntax.CallStatement? syntax ); } - syntax = new Syntax.CallStatement( - callParts[0], - callParts[1] - .Split(',') - .Select(p => p.Trim()) - .Where(p => !string.IsNullOrWhiteSpace(p)) - .ToArray() - ); + var functionName = text.Substring(0, openParen).Trim(); + var paramString = text.Substring(openParen + 1, closeParen - openParen - 1); + if (string.IsNullOrWhiteSpace(functionName)) + { + throw context.Failure( + blockContext.IdentifierIndex, + "{% call %} statement requires a function name and () with optional parameters" + ); + } + + var parameters = ParseParameters(paramString); + syntax = new Syntax.CallStatement(functionName, parameters ?? []); return true; } + private static List? ParseParameters(string paramString) + { + List? parameters = null; + var sb = new StringBuilder(); + var parenDepth = 0; + var inQuotes = false; + var quoteChar = '\0'; + for (var i = 0; i < paramString.Length; i++) + { + var c = paramString[i]; + if (c is '"' or '\'' && (i == 0 || paramString[i - 1] != '\\')) + { + if (!inQuotes) + { + inQuotes = true; + quoteChar = c; + } + else if (quoteChar == c) + { + inQuotes = false; + } + sb.Append(c); + } + else + { + switch (inQuotes) + { + case false when c == '(': + parenDepth++; + sb.Append(c); + break; + case false when c == ')': + parenDepth--; + sb.Append(c); + break; + case false when parenDepth == 0 && c == ',': + AddParameter(); + sb.Clear(); + break; + default: + sb.Append(c); + break; + } + } + } + if (sb.Length > 0) + { + AddParameter(); + } + return parameters; + + void AddParameter() + { + parameters ??= []; + var parameter = sb.ToString().Trim(); + if (!string.IsNullOrWhiteSpace(parameter)) + { + parameters.Add(parameter); + } + } + } + private static void ParseConditionalStatement( Context context, BlockContext blockContext, diff --git a/Cutout/Parser/Syntax.cs b/Cutout/Parser/Syntax.cs index bd4023c..fa90598 100644 --- a/Cutout/Parser/Syntax.cs +++ b/Cutout/Parser/Syntax.cs @@ -16,14 +16,13 @@ internal sealed record RawText(TokenList Value) : Syntax public bool TryGetLeadingWhitespace(out Token? leadingWhitespace) { leadingWhitespace = null; - var count = Value.Count; if (count == 0 || Value[count - 1].Type != TokenType.Whitespace) { return false; } - if (count <= 1 || Value[count - 2].Type != TokenType.Newline) + if (count < 2 || Value[count - 2].Type != TokenType.Newline) { return false; } diff --git a/Cutout/TemplateSourceGenerator.cs b/Cutout/TemplateSourceGenerator.cs index 2859366..c3124d0 100644 --- a/Cutout/TemplateSourceGenerator.cs +++ b/Cutout/TemplateSourceGenerator.cs @@ -11,7 +11,9 @@ namespace Cutout; /// /// Source generator for Cutout templates /// +#pragma warning disable RS1038 [Generator] +#pragma warning restore RS1038 public sealed partial class TemplateSourceGenerator : IIncrementalGenerator { ///