CST: Handle interpolated strings (#239)

This PR adds support for serializing interpolated strings. The structure
of interpolated strings is a bit special compared to other nodes: we
store the string literal parts and the expression parts independently.
The visitor must account for this so that parts are visited in the
correct order.

We implement a special printer for interpolated strings, similar to how
we've implemented it for other strings. We are guaranteed that an
interpolated string starts and ends with a string literal (potentially
empty), so the leading trivia of the first literal and trailing trivia
of the last literal become the trivia for the overall interpolated
string.

Note that the characters `` ` `` , `{` and `}` are not stored anywhere
as part of the CST. To ensure correct trivia attachment to tokens, we
need to ensure that the end position of the string part is computed
correctly. We manually increment current position to do this.

Author: JohnnyMorganz

batteries/syntax/ast_types.luau

@@ -153,6 +153,12 @@ export type AstExprBinary = {
 	rhsoperand: AstExpr,
 }
 
+export type AstExprInterpString = {
+	tag: "interpolatedstring",
+	strings: { Token<string> },
+	expressions: { AstExpr },
+}
+
 export type AstExpr =
 	| AstExprGroup
 	| AstExprConstantNil
@@ -169,6 +175,7 @@ export type AstExpr =
 	| AstExprTable
 	| AstExprUnary
 	| AstExprBinary
+	| AstExprInterpString
 
 export type AstStatBlock = {
 	tag: "block",

batteries/syntax/printer.luau

@@ -46,6 +46,31 @@ local function printString(expr: T.AstExprConstantString): string
 	return result
 end
 
+local function printInterpolatedString(expr: T.AstExprInterpString): string
+	local result = ""
+
+	for i = 1, #expr.strings do
+		result ..= printTriviaList(expr.strings[i].leadingTrivia)
+		if i == 1 then
+			result ..= "`"
+		else
+			result ..= "}"
+		end
+		result ..= expr.strings[i].text
+
+		if i == #expr.strings then
+			result ..= "`"
+			result ..= printTriviaList(expr.strings[i].trailingTrivia)
+		else
+			result ..= "{"
+			result ..= printTriviaList(expr.strings[i].trailingTrivia)
+			result ..= printExpr(expr.expressions[i])
+		end
+	end
+
+	return result
+end
+
 type PrintVisitor = visitor.Visitor & {
 	result: string,
 }
@@ -69,6 +94,11 @@ local function printVisitor()
 		return false
 	end
 
+	printer.visitInterpolatedString = function(node: T.AstExprInterpString)
+		printer.result ..= printInterpolatedString(node)
+		return false
+	end
+
 	return printer
 end
 

batteries/syntax/visitor.luau

@@ -31,6 +31,7 @@ export type Visitor = {
 	visitIndexName: (T.AstExprIndexName) -> boolean,
 	visitIndexExpr: (T.AstExprIndexExpr) -> boolean,
 	visitGroup: (T.AstExprGroup) -> boolean,
+	visitInterpolatedString: (T.AstExprInterpString) -> boolean,
 
 	visitTypeReference: (T.AstTypeReference) -> boolean,
 	visitTypeBoolean: (T.AstTypeSingletonBool) -> boolean,
@@ -80,6 +81,7 @@ local defaultVisitor: Visitor = {
 	visitIndexName = alwaysVisit :: any,
 	visitIndexExpr = alwaysVisit :: any,
 	visitGroup = alwaysVisit :: any,
+	visitInterpolatedString = alwaysVisit,
 
 	visitTypeReference = alwaysVisit :: any,
 	visitTypeBoolean = alwaysVisit :: any,
@@ -404,6 +406,17 @@ local function visitGroup(node: T.AstExprGroup, visitor: Visitor)
 	end
 end
 
+local function visitInterpolatedString(node: T.AstExprInterpString, visitor: Visitor)
+	if visitor.visitInterpolatedString(node) then
+		for i = 1, #node.strings do
+			visitToken(node.strings[i], visitor)
+			if i < #node.expressions then
+				visitExpression(node.expressions[i], visitor)
+			end
+		end
+	end
+end
+
 local function visitTypeReference(node: T.AstTypeReference, visitor: Visitor)
 	if visitor.visitTypeReference(node) then
 		if node.prefix then
@@ -486,6 +499,8 @@ function visitExpression(expression: T.AstExpr, visitor: Visitor)
 			visitIndexExpr(expression, visitor)
 		elseif expression.tag == "group" then
 			visitGroup(expression, visitor)
+		elseif expression.tag == "interpolatedstring" then
+			visitInterpolatedString(expression, visitor)
 		else
 			exhaustiveMatch(expression.tag)
 		end

luau/src/luau.cpp

@@ -979,21 +979,44 @@ struct AstSerialize : public Luau::AstVisitor
 
     void serialize(Luau::AstExprInterpString* node)
     {
+        const auto* cstNode = lookupCstNode<Luau::CstExprInterpString>(node);
+
         lua_rawcheckstack(L, 3);
         lua_createtable(L, 0, preambleSize + 2);
 
         serializeNodePreamble(node, "interpolatedstring");
 
         lua_createtable(L, node->strings.size, 0);
+        lua_createtable(L, node->expressions.size, 0);
+
         for (size_t i = 0; i < node->strings.size; i++)
         {
-            lua_pushlstring(L, node->strings.data[i].data, node->strings.data[i].size);
-            lua_rawseti(L, -2, i + 1);
+            if (cstNode)
+            {
+                auto position = i > 0 ? cstNode->stringPositions.data[i] : node->location.begin;
+                serializeToken(position, std::string(cstNode->sourceStrings.data[i].data, cstNode->sourceStrings.data[i].size).data());
+
+                // Unlike normal tokens, interpolated string parts contain extra characters (`, } or {) that were not included during advancement
+                // For simplicity, lets set the current position manually. We don't have an end position for these parts, so we must compute
+                // If string part was single line, end position = current position + 2 (start and end character)
+                // If string parts was multi line, end position = current position + 1 (just end character)
+                if (position.line == currentPosition.line)
+                    currentPosition.column += 2;
+                else
+                    currentPosition.column += 1;
+            }
+            else
+                lua_pushlstring(L, node->strings.data[i].data, node->strings.data[i].size);
+            lua_rawseti(L, -3, i + 1);
+
+            if (i < node->expressions.size)
+            {
+                node->expressions.data[i]->visit(this);
+                lua_rawseti(L, -2, i + 1);
+            }
         }
+        lua_setfield(L, -3, "expressions");
         lua_setfield(L, -2, "strings");
-
-        serializeExprs(node->expressions);
-        lua_setfield(L, -2, "expressions");
     }
 
     void serialize(Luau::AstExprError* node)

tests/astSerializerTests/interpolated-string-1.luau

@@ -0,0 +1,5 @@
+local a = `simple`
+local b = `start{true}`
+local c = `start{true}end`
+local d = `start \
+	{true} end`

tests/astSerializerTests/interpolated-string-2.luau

@@ -0,0 +1,8 @@
+-- stylua: ignore
+local x =
+	`test`
+
+-- stylua: ignore
+print(
+	`test {subject} with {subject}`
+)

tests/testAstSerializer.spec.luau

@@ -120,13 +120,21 @@ end
 local function test_roundtrippableAst()
 	local files = {
 		"examples/a.luau",
+		"examples/ansi.luau",
 		"examples/async_read.luau",
 		"examples/b.luau",
 		"examples/badisnan.luau",
+		"examples/directories.luau",
 		"examples/json.luau",
 		"examples/main.luau",
 		"examples/net_example.luau",
+		"examples/parallel_serve_helper.luau",
+		"examples/parallel_serve.luau",
+		"examples/parallel_sort_helper.luau",
+		"examples/parallel_sort.luau",
 		"examples/parsing.luau",
+		"examples/process_env.luau",
+		"examples/spawn_example.luau",
 		"examples/time_example.luau",
 		"examples/writeFile.luau",
 		"tests/astSerializerTests/assignment-1.luau",
@@ -135,6 +143,8 @@ local function test_roundtrippableAst()
 		"tests/astSerializerTests/function-declaration-1.luau",
 		"tests/astSerializerTests/function-declaration-2.luau",
 		"tests/astSerializerTests/generic-for-loop-1.luau",
+		"tests/astSerializerTests/interpolated-string-1.luau",
+		"tests/astSerializerTests/interpolated-string-2.luau",
 		"tests/astSerializerTests/local-function-declaration-1.luau",
 		"tests/astSerializerTests/numeric-for-loop-1.luau",
 		"tests/astSerializerTests/type-singletons-1.luau",