Start making the CST usable in Lute (luau-lang#71)

Now that CST nodes exist in Luau, this PR extends the existing
`@lute/luau` C++ module to return full-fidelity CST nodes to Luau,
rather than just AST nodes. This is aimed to support future usages of
the CST for tooling such as code modding, linting or formatting
implemented fully in Luau.

We extend the serialization of some AST nodes with extra tokens, and
create tests to check for validity of tokenization and the
roundtrippability of the serialization state in Luau (i.e.,
`print(parse(source) == source` where `print` is fully defined in Luau).
Future AST nodes will be added in follow up commits.

Alongside the CST data, a key concept that this PR introduces is the
idea of trivia. Right now, the Luau CST does not preserve trivia itself,
but retains enough information to compute spans of trivia, allowing
later lookup in the original source code. To extract trivia, we keep the
original input buffer around, and then take the span between the last
token and the next token, and call this "trivia". Trivia is then
individually tokenized and attached to "Token" nodes.

There are some key rules for trivia:
- [x] Trivia consists of whitespace, single line comments and multi line
comments. Each kind is a separate trivia token
- [x] Whitespace trivia is split up into separate tokens based on the
'\n' character. Other characters of whitespace are joined together up to
and including the first newline character. i.e.,
`<space><space>\n\t\t\n\n` is split into `[<space><space>\n, \t\t\n,
\n]`
- [ ] Trivia on a token is split up into leading and trailing trivia.
Leading / trailing split is based on the Roslyn Compiler trivia rules:
trailing trivia consists of everything up to and including the first
`\n` newline character, and all further trivia is leading trivia of the
next token. The idea being that `local x = ...\n` can then be
represented as a single line in trivia, to allow easier replacement
(rather than the `\n` being part of the next statement).

The last rule is not yet implemented but will be introduced in a
follow-up commit.

The API still remains experimental and is subject to change. In
particular, I'm not fully sold on the naming scheme for nodes and fields
(particularly the fields that match keyword names) and it may require
some standardisation. Also, we are somewhat mixing Ast nodes with Cst
nodes, and we may want to separate the two.

Author: JohnnyMorganz

batteries/syntax/ast_types.luau

@@ -0,0 +1,141 @@
+export type Position = {
+	line: number,
+	column: number,
+}
+
+export type Location = {
+	begin: Position,
+	["end"]: Position, -- TODO: do we really want to have to use brackets everywhere?
+}
+
+export type Whitespace = {
+	tag: "whitespace",
+	location: Location,
+	text: string,
+}
+
+export type SingleLineComment = {
+	tag: "comment",
+	location: Location,
+	text: string,
+}
+
+export type MultiLineComment = {
+	tag: "blockcomment",
+	location: Location,
+	text: string,
+	-- TODO: depth: number,
+}
+
+export type Trivia = Whitespace | SingleLineComment | MultiLineComment
+
+export type Token<Kind = string> = {
+	read leadingTrivia: { Trivia },
+	read position: Position,
+	read text: Kind,
+	read trailingTrivia: { Trivia },
+}
+
+export type Pair<T, Separator> = { node: T, separator: Token<Separator>? }
+export type Punctuated<T, Separator = ","> = { Pair<T, Separator> }
+
+export type AstLocal = {
+	name: Token<string>,
+}
+
+export type AstExprConstantString = Token<string> & {
+	tag: "string",
+	quoteStyle: "single" | "double" | "block" | "interp",
+	blockDepth: number,
+}
+
+export type AstExprLocal = {
+	tag: "local",
+	token: Token<string>,
+	["local"]: AstLocal,
+	upvalue: boolean,
+}
+
+export type AstExprGlobal = {
+	tag: "global",
+	name: Token,
+}
+
+export type AstExprCall = {
+	tag: "call",
+	func: AstExpr, -- TODO: stricter?
+	openParens: Token<"(">?,
+	arguments: Punctuated<AstExpr>,
+	closeParens: Token<")">?,
+}
+
+export type AstExprIndexName = {
+	tag: "indexname",
+	expr: AstExpr,
+	accessor: Token<"." | ":">,
+	index: Token<string>,
+	indexLocation: Location,
+}
+
+export type AstExprIndexExpr = {
+	tag: "index",
+	expr: AstExpr,
+	index: AstExpr,
+}
+
+export type AstExprBinary = {
+	tag: "binary",
+	lhsoperand: AstExpr,
+	operator: Token, -- TODO: enforce token type
+	rhsoperand: AstExpr,
+}
+
+export type AstExprTableItem =
+	| { kind: "list", value: AstExpr }
+	| { kind: "record", key: string, equals: Token<"=">, value: AstExpr }
+	| { kind: "general", key: string, equals: Token<"=">, value: AstExpr }
+
+export type AstExprTable = {
+	tag: "table",
+	openBrace: Token<"{">,
+	entries: { never },
+	closeBrace: Token<"}">,
+}
+
+export type AstExpr =
+	| AstExprConstantString
+	| AstExprLocal
+	| AstExprGlobal
+	| AstExprCall
+	| AstExprIndexName
+	| AstExprIndexExpr
+	| AstExprBinary
+	| AstExprTable
+
+export type AstStatBlock = {
+	tag: "block",
+	statements: { AstStat },
+}
+
+export type AstStatReturn = {
+	tag: "return",
+	["return"]: Token<"return">,
+	expressions: Punctuated<AstExpr>,
+}
+
+export type AstStatExpr = {
+	tag: "expression",
+	expression: AstExpr,
+}
+
+export type AstStatLocal = {
+	tag: "local",
+	["local"]: Token<"local">,
+	variables: Punctuated<AstLocal>,
+	equals: Token<"=">?,
+	values: Punctuated<AstExpr>,
+}
+
+export type AstStat = AstStatBlock | AstStatReturn | AstStatExpr | AstStatLocal
+
+return {}

batteries/syntax/parser.luau

@@ -0,0 +1,13 @@
+--!strict
+
+local luau = require("@lute/luau")
+local T = require("./ast_types")
+
+--- Parses Luau source code into an AstStatBlock
+local function parse(source: string): T.AstStatBlock
+	return luau.parse(source).root
+end
+
+return {
+	parse = parse,
+}

batteries/syntax/printer.luau

@@ -0,0 +1,71 @@
+--!strict
+local T = require("./ast_types")
+local visitor = require("./visitor")
+
+local function exhaustiveMatch(value: never): never
+	error(`Unknown value in exhaustive match: {value}`)
+end
+
+local function printTrivia(trivia: T.Trivia): string
+	if trivia.tag == "whitespace" or trivia.tag == "comment" or trivia.tag == "blockcomment" then
+		return trivia.text
+	else
+		return exhaustiveMatch(trivia.tag)
+	end
+end
+
+local function printTriviaList(trivia: { T.Trivia })
+	local result = ""
+	for _, trivia in trivia do
+		result ..= printTrivia(trivia)
+	end
+	return result
+end
+
+local function printToken(token: T.Token): string
+	return printTriviaList(token.leadingTrivia) .. token.text .. printTriviaList(token.trailingTrivia)
+end
+
+local function printString(expr: T.AstExprConstantString): string
+	local result = printTriviaList(expr.leadingTrivia)
+
+	if expr.quoteStyle == "single" then
+		result ..= `'{expr.text}'`
+	elseif expr.quoteStyle == "double" then
+		result ..= `"{expr.text}"`
+	elseif expr.quoteStyle == "block" then
+		local equals = string.rep("=", expr.blockDepth)
+		result ..= `[{equals}[{expr.text}]{equals}]`
+	elseif expr.quoteStyle == "interp" then
+		result ..= "`" .. expr.text .. "`"
+	else
+		return exhaustiveMatch(expr.quoteStyle)
+	end
+
+	result ..= printTriviaList(expr.trailingTrivia)
+	return result
+end
+
+--- Returns a string representation of an AstStatBlock
+local function printBlock(block: T.AstStatBlock): string
+	local printer = visitor.createVisitor()
+	local result = ""
+
+	printer.visitToken = function(node: T.Token)
+		result ..= printToken(node)
+		return false
+	end
+
+	printer.visitString = function(node: T.AstExprConstantString)
+		result ..= printString(node)
+		return false
+	end
+
+	visitor.visitBlock(block, printer)
+
+	return result
+end
+
+return {
+	print = printBlock,
+}