Skip to content

Commit fafb5c5

Browse files
JohnnyMorganzJohnnyMorganz
authored
CST: Fix colon positions and minor improvements (#284)
- Use the real positions of colons (now available from new Luau version) - Serialize value of `self` on an AstExprCall - Fix some size expectations in `lua_rawcheckstack` and `lua_createtable` - Add a visitor for end of a local declaration - Simplify testing structure and run more tests
1 parent 694ea37 commit fafb5c5

File tree

5 files changed

+47
-87
lines changed

5 files changed

+47
-87
lines changed

‎batteries/syntax/ast_types.luau‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ export type AstExprCall = {
8686
openParens: Token<"(">?,
8787
arguments: Punctuated<AstExpr>,
8888
closeParens: Token<")">?,
89+
self: boolean,
8990
}
9091

9192
export type AstExprIndexName = {

‎batteries/syntax/visitor.luau‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ export type Visitor = {
1010
visitRepeat: (T.AstStatRepeat) -> boolean,
1111
visitReturn: (T.AstStatReturn) -> boolean,
1212
visitLocalDeclaration: (T.AstStatLocal) -> boolean,
13+
visitLocalDeclarationEnd: (T.AstStatLocal) -> (),
1314
visitFor: (T.AstStatFor) -> boolean,
1415
visitForIn: (T.AstStatForIn) -> boolean,
1516
visitAssign: (T.AstStatAssign) -> boolean,
@@ -72,6 +73,7 @@ local defaultVisitor: Visitor = {
7273
visitRepeat = alwaysVisit :: any,
7374
visitReturn = alwaysVisit :: any,
7475
visitLocalDeclaration = alwaysVisit :: any,
76+
visitLocalDeclarationEnd = alwaysVisit :: any,
7577
visitFor = alwaysVisit :: any,
7678
visitForIn = alwaysVisit :: any,
7779
visitAssign = alwaysVisit :: any,
@@ -217,6 +219,8 @@ local function visitLocalStatement(node: T.AstStatLocal, visitor: Visitor)
217219
visitToken(node.equals, visitor)
218220
end
219221
visitPunctuated(node.values, visitor, visitExpression)
222+
223+
visitor.visitLocalDeclarationEnd(node)
220224
end
221225
end
222226

‎luau/src/luau.cpp‎

Lines changed: 26 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ const char* COMPILE_RESULT_TYPE = "CompileResult";
2222
LUAU_FASTFLAG(LuauStoreCSTData2)
2323
LUAU_FASTFLAG(LuauFixFunctionWithAttributesStartLocation)
2424
LUAU_FASTFLAG(LuauStoreReturnTypesAsPackOnAst)
25+
LUAU_FASTFLAG(LuauStoreLocalAnnotationColonPositions)
2526

2627
namespace luau
2728
{
@@ -40,6 +41,7 @@ static StatResult parse(std::string& source)
4041
FFlag::LuauStoreCSTData2.value = true;
4142
FFlag::LuauFixFunctionWithAttributesStartLocation.value = true;
4243
FFlag::LuauStoreReturnTypesAsPackOnAst.value = true;
44+
FFlag::LuauStoreLocalAnnotationColonPositions.value = true;
4345

4446
auto allocator = std::make_shared<Luau::Allocator>();
4547
auto names = std::make_shared<Luau::AstNameTable>(*allocator);
@@ -68,6 +70,7 @@ static ExprResult parseExpr(std::string& source)
6870
FFlag::LuauStoreCSTData2.value = true;
6971
FFlag::LuauFixFunctionWithAttributesStartLocation.value = true;
7072
FFlag::LuauStoreReturnTypesAsPackOnAst.value = true;
73+
FFlag::LuauStoreLocalAnnotationColonPositions.value = true;
7174

7275
auto allocator = std::make_shared<Luau::Allocator>();
7376
auto names = std::make_shared<Luau::AstNameTable>(*allocator);
@@ -320,7 +323,7 @@ struct AstSerialize : public Luau::AstVisitor
320323
lua_pushstring(L, name.value);
321324
}
322325

323-
void serialize(Luau::AstLocal* local, bool createToken = true)
326+
void serialize(Luau::AstLocal* local, bool createToken = true, std::optional<Luau::Position> colonPosition = std::nullopt)
324327
{
325328
lua_rawcheckstack(L, 2);
326329

@@ -330,7 +333,7 @@ struct AstSerialize : public Luau::AstVisitor
330333
if (lua_isnil(L, -1))
331334
{
332335
lua_pop(L, 1);
333-
lua_createtable(L, 0, 3);
336+
lua_createtable(L, 0, 4);
334337

335338
// set up reference for this local into the local table
336339
lua_pushlightuserdata(L, local);
@@ -344,7 +347,10 @@ struct AstSerialize : public Luau::AstVisitor
344347

345348
// TODO: get position of colon properly
346349
if (local->annotation)
347-
serializeToken(currentPosition, ":");
350+
{
351+
LUAU_ASSERT(colonPosition);
352+
serializeToken(*colonPosition, ":");
353+
}
348354
else
349355
lua_pushnil(L);
350356
lua_setfield(L, -2, "colon");
@@ -464,12 +470,11 @@ struct AstSerialize : public Luau::AstVisitor
464470

465471
void serializeTrivia(const std::vector<Trivia>& trivia)
466472
{
467-
lua_rawcheckstack(L, 2);
473+
lua_rawcheckstack(L, 3);
468474
lua_createtable(L, trivia.size(), 0);
469475

470476
for (size_t i = 0; i < trivia.size(); i++)
471477
{
472-
lua_rawcheckstack(L, 2);
473478
lua_createtable(L, 0, 3);
474479

475480
switch (trivia[i].kind)
@@ -499,7 +504,7 @@ struct AstSerialize : public Luau::AstVisitor
499504
// For correct trivia computation, everything must end up going through serializeToken
500505
void serializeToken(Luau::Position position, const char* text, int nrec = 0)
501506
{
502-
lua_rawcheckstack(L, 2);
507+
lua_rawcheckstack(L, 3);
503508
lua_createtable(L, 0, nrec + 3);
504509

505510
const auto trivia = extractTrivia(position);
@@ -532,7 +537,6 @@ struct AstSerialize : public Luau::AstVisitor
532537
lua_setfield(L, -2, "text");
533538
advancePosition(text);
534539

535-
lua_rawcheckstack(L, 2);
536540
lua_createtable(L, 0, 0);
537541
lua_setfield(L, -2, "trailingTrivia");
538542

@@ -590,12 +594,11 @@ struct AstSerialize : public Luau::AstVisitor
590594
template<typename T>
591595
void serializePunctuated(Luau::AstArray<T> nodes, Luau::AstArray<Luau::Position> separators, const char* separatorText)
592596
{
593-
lua_rawcheckstack(L, 2);
597+
lua_rawcheckstack(L, 3);
594598
lua_createtable(L, nodes.size, 0);
595599

596600
for (size_t i = 0; i < nodes.size; i++)
597601
{
598-
lua_rawcheckstack(L, 2);
599602
lua_createtable(L, 0, 2);
600603

601604
nodes.data[i]->visit(this);
@@ -637,17 +640,16 @@ struct AstSerialize : public Luau::AstVisitor
637640
}
638641
}
639642

640-
void serializePunctuated(Luau::AstArray<Luau::AstLocal*> nodes, Luau::AstArray<Luau::Position> separators, const char* separatorText)
643+
void serializePunctuated(Luau::AstArray<Luau::AstLocal*> nodes, Luau::AstArray<Luau::Position> separators, const char* separatorText, Luau::AstArray<Luau::Position> colonPositions)
641644
{
642-
lua_rawcheckstack(L, 2);
645+
lua_rawcheckstack(L, 3);
643646
lua_createtable(L, nodes.size, 0);
644647

645648
for (size_t i = 0; i < nodes.size; i++)
646649
{
647-
lua_rawcheckstack(L, 2);
648650
lua_createtable(L, 0, 2);
649651

650-
serialize(nodes.data[i]);
652+
serialize(nodes.data[i], /* createToken=*/ true, colonPositions.size > i ? std::make_optional(colonPositions.data[i]): std::nullopt);
651653
lua_setfield(L, -2, "node");
652654

653655
if (i < separators.size)
@@ -777,7 +779,7 @@ struct AstSerialize : public Luau::AstVisitor
777779
serializeToken(node->location.begin, node->local->name.value);
778780
lua_setfield(L, -2, "token"),
779781

780-
serialize(node->local);
782+
serialize(node->local);
781783
lua_setfield(L, -2, "local");
782784

783785
lua_pushboolean(L, node->upvalue);
@@ -822,6 +824,9 @@ struct AstSerialize : public Luau::AstVisitor
822824
serializePunctuated(node->args, cstNode ? cstNode->commaPositions : Luau::AstArray<Luau::Position>{}, ",");
823825
lua_setfield(L, -2, "arguments");
824826

827+
lua_pushboolean(L, node->self);
828+
lua_setfield(L, -2, "self");
829+
825830
serialize(node->argLocation);
826831
lua_setfield(L, -2, "argLocation");
827832

@@ -884,7 +889,7 @@ struct AstSerialize : public Luau::AstVisitor
884889
const auto* cstNode = lookupCstNode<Luau::CstExprFunction>(node);
885890

886891
lua_rawcheckstack(L, 3);
887-
lua_createtable(L, 0, 9);
892+
lua_createtable(L, 0, 14);
888893

889894
if (node->generics.size > 0 || node->genericPacks.size > 0)
890895
{
@@ -920,7 +925,7 @@ struct AstSerialize : public Luau::AstVisitor
920925
lua_setfield(L, -2, "openParens");
921926
}
922927

923-
serializePunctuated(node->args, cstNode ? cstNode->argsCommaPositions : Luau::AstArray<Luau::Position>{}, ",");
928+
serializePunctuated(node->args, cstNode ? cstNode->argsCommaPositions : Luau::AstArray<Luau::Position>{}, ",", cstNode ? cstNode->argsAnnotationColonPositions : Luau::AstArray<Luau::Position>{});
924929
lua_setfield(L, -2, "parameters");
925930

926931
if (node->vararg)
@@ -929,9 +934,8 @@ struct AstSerialize : public Luau::AstVisitor
929934
lua_pushnil(L);
930935
lua_setfield(L, -2, "vararg");
931936

932-
// TODO: get proper position of colon
933-
if (node->varargAnnotation)
934-
serializeToken(currentPosition, ":");
937+
if (cstNode && node->varargAnnotation)
938+
serializeToken(cstNode->varargAnnotationColonPosition, ":");
935939
else
936940
lua_pushnil(L);
937941
lua_setfield(L, -2, "varargColon");
@@ -1384,7 +1388,7 @@ struct AstSerialize : public Luau::AstVisitor
13841388
lua_setfield(L, -2, "local");
13851389

13861390
const auto cstNode = lookupCstNode<Luau::CstStatLocal>(node);
1387-
serializePunctuated(node->vars, cstNode ? cstNode->varsCommaPositions : Luau::AstArray<Luau::Position>{}, ",");
1391+
serializePunctuated(node->vars, cstNode ? cstNode->varsCommaPositions : Luau::AstArray<Luau::Position>{}, ",", cstNode ? cstNode->varsAnnotationColonPositions : Luau::AstArray<Luau::Position>{});
13881392
lua_setfield(L, -2, "variables");
13891393

13901394
if (node->equalsSignLocation)
@@ -1409,7 +1413,7 @@ struct AstSerialize : public Luau::AstVisitor
14091413
serializeToken(node->location.begin, "for");
14101414
lua_setfield(L, -2, "for");
14111415

1412-
serialize(node->var);
1416+
serialize(node->var, /* createToken= */ true, cstNode ? std::make_optional(cstNode->annotationColonPosition) : std::nullopt);
14131417
lua_setfield(L, -2, "variable");
14141418

14151419
if (cstNode)
@@ -1467,7 +1471,7 @@ struct AstSerialize : public Luau::AstVisitor
14671471
serializeToken(node->location.begin, "for");
14681472
lua_setfield(L, -2, "for");
14691473

1470-
serializePunctuated(node->vars, cstNode ? cstNode->varsCommaPositions : Luau::AstArray<Luau::Position>{}, ",");
1474+
serializePunctuated(node->vars, cstNode ? cstNode->varsCommaPositions : Luau::AstArray<Luau::Position>{}, ",", cstNode ? cstNode->varsAnnotationColonPositions : Luau::AstArray<Luau::Position>{});
14711475
lua_setfield(L, -2, "variables");
14721476

14731477
if (node->hasIn)

‎tests/astSerializerTests/local-assignment-1.luau‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
local a = 1
2+
local b, c = 2, nil
3+
local d: string = ""

‎tests/testAstSerializer.spec.luau‎

Lines changed: 13 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -118,72 +118,20 @@ local function test_triviaSplitBetweenLeadingAndTrailing()
118118
end
119119

120120
local function test_roundtrippableAst()
121-
local files = {
122-
"examples/a.luau",
123-
"examples/ansi.luau",
124-
"examples/async_read.luau",
125-
"examples/b.luau",
126-
"examples/badisnan.luau",
127-
-- "examples/cliargs.luau", genuine failure
128-
"examples/compile.luau",
129-
"examples/directories.luau",
130-
"examples/json.luau",
131-
"examples/main.luau",
132-
"examples/net_example.luau",
133-
"examples/parallel_serve_helper.luau",
134-
"examples/parallel_serve.luau",
135-
"examples/parallel_sort_helper.luau",
136-
"examples/parallel_sort.luau",
137-
"examples/parsing.luau",
138-
"examples/process_env.luau",
139-
-- "examples/process.luau", genuine failure (table?)
140-
-- "examples/serve_html.luau", genuine failure (advancing)
141-
"examples/serve.luau",
142-
"examples/spawn_example.luau",
143-
"examples/time_example.luau",
144-
-- "examples/toml.luau", genuine failure
145-
"examples/writeFile.luau",
146-
"tests/astSerializerTests/assignment-1.luau",
147-
"tests/astSerializerTests/attributes-1.luau",
148-
"tests/astSerializerTests/break-continue-1.luau",
149-
"tests/astSerializerTests/compound-assignment-1.luau",
150-
"tests/astSerializerTests/compound-types-1.luau",
151-
"tests/astSerializerTests/compound-types-2.luau",
152-
"tests/astSerializerTests/compound-types-3.luau",
153-
"tests/astSerializerTests/function-declaration-1.luau",
154-
"tests/astSerializerTests/function-declaration-2.luau",
155-
"tests/astSerializerTests/function-declaration-3.luau",
156-
"tests/astSerializerTests/function-declaration-4.luau",
157-
"tests/astSerializerTests/function-type-1.luau",
158-
"tests/astSerializerTests/function-type-2.luau",
159-
"tests/astSerializerTests/generic-for-loop-1.luau",
160-
"tests/astSerializerTests/if-expression-1.luau",
161-
"tests/astSerializerTests/if-expression-2.luau",
162-
"tests/astSerializerTests/if-statement-1.luau",
163-
"tests/astSerializerTests/interpolated-string-1.luau",
164-
"tests/astSerializerTests/interpolated-string-2.luau",
165-
"tests/astSerializerTests/local-function-declaration-1.luau",
166-
"tests/astSerializerTests/numeric-for-loop-1.luau",
167-
"tests/astSerializerTests/table-1.luau",
168-
"tests/astSerializerTests/table-2.luau",
169-
"tests/astSerializerTests/type-alias-1.luau",
170-
"tests/astSerializerTests/type-assertion-1.luau",
171-
"tests/astSerializerTests/type-function-1.luau",
172-
"tests/astSerializerTests/type-singletons-1.luau",
173-
"tests/astSerializerTests/type-tables-1.luau",
174-
"tests/astSerializerTests/type-tables-2.luau",
175-
"tests/astSerializerTests/while-1.luau",
176-
"tests/astSerializerTests/repeat-until-1.luau",
177-
}
178-
179-
for _, path in files do
180-
print(path)
181-
local source = fs.readfiletostring(path)
182-
local result = printer.printfile(parser.parsefile(source))
183-
184-
print(result)
185-
assert(source == result)
121+
local function visitDirectory(directory: string)
122+
for _, entry in fs.listdir(directory) do
123+
local path = `{directory}/{entry.name}`
124+
print(path)
125+
local source = fs.readfiletostring(path)
126+
local result = printer.printfile(parser.parsefile(source))
127+
128+
print(result)
129+
assert(source == result)
130+
end
186131
end
132+
133+
visitDirectory("examples")
134+
visitDirectory("tests/astSerializerTests")
187135
end
188136

189137
test_tokenContainsLeadingSpaces()

0 commit comments

Comments
 (0)
⚡