tokenize.zig

//! Tokenization functionality for the Roc parser.
//!
//! This module provides the tokenizer that converts Roc source code into
//! a stream of tokens for parsing. It handles all Roc language tokens including
//! keywords, identifiers, literals, operators, and punctuation, representing
//! them as offsets into the source code with additional metadata.

const std = @import("std");
const Allocator = std.mem.Allocator;
const base = @import("base");
const tracy = @import("tracy");

const DataSpan = base.DataSpan;
const CommonEnv = base.CommonEnv;

/// representation of a token in the source code, like '+', 'foo', '=', '{'
/// these are represented by an offset into the bytes of the source code
/// and an extra field that stores either the length of the token or
/// an index into the string interner
pub const Token = struct {
    tag: Tag,
    region: base.Region,
    extra: Extra,

    pub const Extra = union {
        interned: base.Ident.Idx,
        ident_with_flags: IdentWithFlags,
        none: u32,
    };

    pub const IdentWithFlags = struct {
        ident: base.Ident.Idx,
        starts_with_underscore: bool,
        ends_with_underscore: bool,
    };

    pub const List = std.MultiArrayList(@This());

    pub const Idx = u32;
    pub const Span = struct { span: DataSpan };

    pub const Tag = enum(u8) {
        EndOfFile,

        // primitives
        Float,
        StringStart, // the " that starts a string
        StringEnd, // the " that ends a string
        MultilineStringStart, // the """ or \\ that starts a multiline string
        StringPart,
        MalformedStringPart, // malformed, but should be treated similar to a StringPart in the parser
        SingleQuote,
        MalformedSingleQuote, // malformed, but should be treated similar to a SingleQuote in the parser
        Int,
        MalformedNumberBadSuffix, // malformed, but should be treated similar to an int in the parser
        MalformedNumberUnicodeSuffix, // malformed, but should be treated similar to an int in the parser
        MalformedNumberNoDigits, // malformed, but should be treated similar to an int in the parser
        MalformedNumberNoExponentDigits, // malformed, but should be treated similar to an int in the parser

        // Should be treated as StringPart in the parser, but we forward the error to the ast
        MalformedInvalidUnicodeEscapeSequence,
        MalformedInvalidEscapeSequence,

        UpperIdent,
        LowerIdent,
        MalformedUnicodeIdent,
        Underscore,
        DotLowerIdent,
        DotInt,
        DotUpperIdent,
        NoSpaceDotInt,
        NoSpaceDotLowerIdent,
        NoSpaceDotUpperIdent,
        MalformedDotUnicodeIdent,
        MalformedNoSpaceDotUnicodeIdent,

        NamedUnderscore,
        MalformedNamedUnderscoreUnicode,

        OpaqueName,
        MalformedOpaqueNameUnicode,
        MalformedOpaqueNameWithoutName,

        OpenRound,
        CloseRound,
        OpenSquare,
        CloseSquare,
        OpenCurly,
        CloseCurly,
        OpenStringInterpolation,
        CloseStringInterpolation,
        NoSpaceOpenRound,
        // NoSpaceOpenCurly,

        OpPlus,
        OpStar,
        OpPizza,
        OpAssign,
        OpBinaryMinus, // trailing whitespace
        OpUnaryMinus, // no trailing whitespace
        OpNotEquals,
        OpBang,
        OpAnd,
        OpAmpersand,
        OpQuestion,
        OpDoubleQuestion,
        OpOr,
        OpBar,
        OpDoubleSlash,
        OpSlash,
        OpPercent,
        OpCaret,
        OpGreaterThanOrEq,
        OpGreaterThan,
        OpLessThanOrEq,
        OpBackArrow,
        OpLessThan,
        OpEquals,
        OpColonEqual,
        OpDoubleColon,
        NoSpaceOpQuestion,

        Comma,
        Dot,
        DoubleDot,
        TripleDot,
        DotStar,
        OpColon,
        OpArrow,
        OpFatArrow,
        OpBackslash,

        // Keywords
        KwApp,
        KwAs,
        KwCrash,
        KwDbg,
        KwElse,
        KwExpect,
        KwExposes,
        KwExposing,
        KwFor,
        KwGenerates,
        KwHas,
        KwHosted,
        KwIf,
        KwImplements,
        KwImport,
        KwImports,
        KwIn,
        KwInterface,
        KwMatch,
        KwModule,
        KwPackage,
        KwPackages,
        KwPlatform,
        KwProvides,
        KwRequires,
        KwReturn,
        KwTargets,
        KwVar,
        KwWhere,
        KwWhile,
        KwWith,
        KwBreak,

        MalformedUnknownToken,

        /// Returns true if the node is malformed.
        pub fn isMalformed(tok: Tag) bool {
            // This function explicitly lists all variants to ensure new malformed nodes aren't missed.
            return switch (tok) {
                .EndOfFile,
                .Float,
                .StringStart,
                .StringEnd,
                .MultilineStringStart,
                .StringPart,
                .SingleQuote,
                .Int,
                .UpperIdent,
                .LowerIdent,
                .Underscore,
                .DotLowerIdent,
                .DotInt,
                .DotUpperIdent,
                .NoSpaceDotInt,
                .NoSpaceDotLowerIdent,
                .NoSpaceDotUpperIdent,
                .NamedUnderscore,
                .OpaqueName,
                .OpenRound,
                .CloseRound,
                .OpenSquare,
                .CloseSquare,
                .OpenCurly,
                .CloseCurly,
                .OpenStringInterpolation,
                .CloseStringInterpolation,
                .NoSpaceOpenRound,
                .OpPlus,
                .OpStar,
                .OpPizza,
                .OpAssign,
                .OpBinaryMinus,
                .OpUnaryMinus,
                .OpNotEquals,
                .OpBang,
                .OpAnd,
                .OpAmpersand,
                .OpQuestion,
                .OpDoubleQuestion,
                .OpOr,
                .OpBar,
                .OpDoubleSlash,
                .OpSlash,
                .OpPercent,
                .OpCaret,
                .OpGreaterThanOrEq,
                .OpGreaterThan,
                .OpLessThanOrEq,
                .OpBackArrow,
                .OpLessThan,
                .OpEquals,
                .OpColonEqual,
                .OpDoubleColon,
                .NoSpaceOpQuestion,
                .Comma,
                .Dot,
                .DoubleDot,
                .TripleDot,
                .DotStar,
                .OpColon,
                .OpArrow,
                .OpFatArrow,
                .OpBackslash,
                .KwApp,
                .KwAs,
                .KwCrash,
                .KwDbg,
                .KwElse,
                .KwExpect,
                .KwExposes,
                .KwExposing,
                .KwFor,
                .KwGenerates,
                .KwHas,
                .KwHosted,
                .KwIf,
                .KwImplements,
                .KwImport,
                .KwImports,
                .KwIn,
                .KwInterface,
                .KwMatch,
                .KwModule,
                .KwPackage,
                .KwPackages,
                .KwPlatform,
                .KwProvides,
                .KwRequires,
                .KwReturn,
                .KwTargets,
                .KwVar,
                .KwWhere,
                .KwWhile,
                .KwWith,
                .KwBreak,
                => false,

                .MalformedDotUnicodeIdent,
                .MalformedInvalidEscapeSequence,
                .MalformedInvalidUnicodeEscapeSequence,
                .MalformedNamedUnderscoreUnicode,
                .MalformedNoSpaceDotUnicodeIdent,
                .MalformedNumberBadSuffix,
                .MalformedNumberNoDigits,
                .MalformedNumberNoExponentDigits,
                .MalformedNumberUnicodeSuffix,
                .MalformedOpaqueNameUnicode,
                .MalformedOpaqueNameWithoutName,
                .MalformedUnicodeIdent,
                .MalformedUnknownToken,
                .MalformedSingleQuote,
                .MalformedStringPart,
                => true,
            };
        }

        pub fn isInterned(tok: Tag) bool {
            return switch (tok) {
                .UpperIdent,
                .LowerIdent,
                .DotLowerIdent,
                .DotUpperIdent,
                .NoSpaceDotLowerIdent,
                .NoSpaceDotUpperIdent,
                .NamedUnderscore,
                .MalformedNamedUnderscoreUnicode,
                .MalformedNoSpaceDotUnicodeIdent,
                .MalformedUnicodeIdent,
                .MalformedDotUnicodeIdent,
                .MalformedOpaqueNameUnicode,
                .OpaqueName,
                => true,
                else => false,
            };
        }

        pub fn hasUnderscoreFlags(tok: Tag) bool {
            return switch (tok) {
                .LowerIdent,
                .NamedUnderscore,
                => true,
                else => false,
            };
        }

        /// Returns true if this token can end an expression, meaning a following
        /// minus sign should be treated as a binary operator rather than unary.
        /// For example, in `x-1`, the minus after `x` (LowerIdent) should be binary.
        pub fn canEndExpression(tok: Tag) bool {
            return switch (tok) {
                // Identifiers can end expressions
                .LowerIdent,
                .UpperIdent,
                .MalformedUnicodeIdent,
                .NamedUnderscore,
                .MalformedNamedUnderscoreUnicode,
                .OpaqueName,
                .MalformedOpaqueNameUnicode,
                // Dot access can end expressions
                .DotLowerIdent,
                .DotUpperIdent,
                .DotInt,
                .NoSpaceDotLowerIdent,
                .NoSpaceDotUpperIdent,
                .NoSpaceDotInt,
                .MalformedDotUnicodeIdent,
                .MalformedNoSpaceDotUnicodeIdent,
                // Numbers can end expressions
                .Int,
                .Float,
                .MalformedNumberBadSuffix,
                .MalformedNumberUnicodeSuffix,
                .MalformedNumberNoDigits,
                .MalformedNumberNoExponentDigits,
                // Closing brackets can end expressions
                .CloseRound,
                .CloseSquare,
                .CloseCurly,
                .CloseStringInterpolation,
                // String literals can end expressions
                .StringEnd,
                .SingleQuote,
                .MalformedSingleQuote,
                => true,
                else => false,
            };
        }

        /// This function is used to keep around the first malformed node.
        /// For example, if an integer has no digits and a bad suffix `0bu22`,
        /// we keep the first malformed node that the integer has no digits instead of pointing out the bad suffix.
        fn updateIfNotMalformed(tok: Tag, next: Tag) Tag {
            if (tok.isMalformed()) {
                return tok;
            }
            return next;
        }
    };

    pub const keywords = std.StaticStringMap(Tag).initComptime(.{
        .{ "and", .OpAnd },
        .{ "app", .KwApp },
        .{ "as", .KwAs },
        .{ "crash", .KwCrash },
        .{ "dbg", .KwDbg },
        .{ "else", .KwElse },
        .{ "expect", .KwExpect },
        .{ "exposes", .KwExposes },
        .{ "exposing", .KwExposing },
        .{ "for", .KwFor },
        .{ "generates", .KwGenerates },
        .{ "has", .KwHas },
        .{ "hosted", .KwHosted },
        .{ "if", .KwIf },
        .{ "implements", .KwImplements },
        .{ "import", .KwImport },
        .{ "imports", .KwImports },
        .{ "in", .KwIn },
        .{ "interface", .KwInterface },
        .{ "match", .KwMatch },
        .{ "module", .KwModule },
        .{ "or", .OpOr },
        .{ "package", .KwPackage },
        .{ "packages", .KwPackages },
        .{ "platform", .KwPlatform },
        .{ "provides", .KwProvides },
        .{ "requires", .KwRequires },
        .{ "return", .KwReturn },
        .{ "targets", .KwTargets },
        .{ "var", .KwVar },
        .{ "where", .KwWhere },
        .{ "while", .KwWhile },
        .{ "with", .KwWith },
        .{ "break", .KwBreak },
    });

    pub const valid_number_suffixes = std.StaticStringMap(void).initComptime(.{
        .{ "dec", .{} },
        .{ "f32", .{} },
        .{ "f64", .{} },
        .{ "i128", .{} },
        .{ "i16", .{} },
        .{ "i32", .{} },
        .{ "i64", .{} },
        .{ "i8", .{} },
        .{ "nat", .{} },
        .{ "u128", .{} },
        .{ "u16", .{} },
        .{ "u32", .{} },
        .{ "u64", .{} },
        .{ "u8", .{} },
    });
};

/// The buffer that accumulates tokens.
pub const TokenizedBuffer = struct {
    tokens: Token.List,
    env: *CommonEnv,

    pub fn initCapacity(env: *CommonEnv, gpa: std.mem.Allocator, capacity: usize) std.mem.Allocator.Error!TokenizedBuffer {
        var tokens = Token.List{};
        try tokens.ensureTotalCapacity(gpa, capacity);
        return TokenizedBuffer{
            .tokens = tokens,
            .env = env,
        };
    }

    pub fn deinit(self: *TokenizedBuffer, gpa: std.mem.Allocator) void {
        self.tokens.deinit(gpa);
    }

    pub fn resolve(self: *const TokenizedBuffer, idx: usize) base.Region {
        return self.tokens.items(.region)[idx];
    }

    /// Loads the current token if it is an identifier.
    /// Otherwise returns null.
    pub fn resolveIdentifier(self: *const TokenizedBuffer, token: Token.Idx) ?base.Ident.Idx {
        const tag = self.tokens.items(.tag)[@intCast(token)];
        const extra = self.tokens.items(.extra)[@intCast(token)];
        if (tag.hasUnderscoreFlags()) {
            return extra.ident_with_flags.ident;
        } else if (tag.isInterned()) {
            return extra.interned;
        } else {
            return null;
        }
    }

    /// Gets underscore flags for identifier tokens.
    /// Returns null if token is not an identifier with underscore flags.
    pub fn resolveUnderscoreFlags(self: *TokenizedBuffer, token: Token.Idx) ?struct { starts_with_underscore: bool, ends_with_underscore: bool } {
        const tag = self.tokens.items(.tag)[@intCast(token)];
        if (tag.hasUnderscoreFlags()) {
            const extra = self.tokens.items(.extra)[@intCast(token)];
            return .{
                .starts_with_underscore = extra.ident_with_flags.starts_with_underscore,
                .ends_with_underscore = extra.ident_with_flags.ends_with_underscore,
            };
        } else {
            return null;
        }
    }
};

/// Represents a diagnostic message including its position in the source.
pub const Diagnostic = struct {
    tag: Tag,
    region: base.Region,

    /// Represents the type of diagnostic message.
    pub const Tag = enum {
        MisplacedCarriageReturn,
        AsciiControl,
        LeadingZero,
        UppercaseBase,
        InvalidUnicodeEscapeSequence,
        InvalidEscapeSequence,
        UnclosedString,
        NonPrintableUnicodeInStrLiteral,
        InvalidUtf8InSource,
        DollarInMiddleOfIdentifier,
        SingleQuoteTooLong,
        SingleQuoteEmpty,
        SingleQuoteUnclosed,
    };
};

/// The cursor is our current position in the input text, and it collects messages.
/// Note that instead of allocating its own message list, the caller must pass in a pre-allocated
/// slice of Message. The field `message_count` tracks how many messages have been written.
/// This can grow beyond the length of the slice, and if so, it means there are more messages
/// than the caller has allocated space for. The caller can either ignore these messages or
/// allocate a larger slice and tokenize again.
pub const Cursor = struct {
    buf: []const u8,
    pos: u32,
    messages: []Diagnostic,
    message_count: u32,
    tab_width: u8 = 4, // TODO: make this configurable

    /// Initialize a Cursor with the given input buffer and a pre-allocated messages slice.
    pub fn init(buf: []const u8, messages: []Diagnostic) Cursor {
        return Cursor{
            .buf = buf,
            .pos = 0,
            .messages = messages,
            .message_count = 0,
        };
    }

    fn pushMessageHere(self: *Cursor, tag: Diagnostic.Tag) void {
        self.pushMessage(tag, self.pos, self.pos);
    }

    fn pushMessage(self: *Cursor, tag: Diagnostic.Tag, begin: u32, end: u32) void {
        if (self.message_count < self.messages.len) {
            self.messages[self.message_count] = Diagnostic{
                .tag = tag,
                .region = base.Region.from_raw_offsets(begin, end),
            };
        }
        self.message_count += 1;
    }

    /// Returns the current byte, or null if at the end.
    pub fn peek(self: *Cursor) ?u8 {
        if (self.pos < self.buf.len) {
            return self.buf[self.pos];
        }
        return null;
    }

    /// Returns the byte at the given lookahead offset.
    pub fn peekAt(self: *Cursor, lookahead: u32) ?u8 {
        if (self.pos + lookahead < self.buf.len) {
            return self.buf[self.pos + lookahead];
        }
        return null;
    }

    pub fn isPeekedCharInRange(self: *Cursor, lookahead: u32, start: u8, end: u8) bool {
        const peeked = self.peekAt(lookahead);

        return if (peeked) |c|
            c >= start and c <= end
        else
            false;
    }

    /// Requires that the next byte is `ch`, otherwise pushes a message.
    pub fn require(self: *Cursor, ch: u8, tag: Diagnostic.Tag) void {
        if (self.peek() == ch) {
            self.pos += 1;
        } else {
            self.pushMessageHere(tag);
        }
    }

    /// Chomps "trivia" (whitespace, comments, etc.).
    pub fn chompTrivia(self: *Cursor) void {
        while (self.pos < self.buf.len) {
            const b = self.buf[self.pos];
            if (b == ' ') {
                self.pos += 1;
            } else if (b == '\t') {
                self.pos += 1;
            } else if (b == '\n') {
                self.pos += 1;
            } else if (b == '\r') {
                self.pos += 1;
                if (self.pos < self.buf.len and self.buf[self.pos] == '\n') {
                    self.pos += 1;
                } else {
                    self.pushMessageHere(.MisplacedCarriageReturn);
                }
            } else if (b == '#') {
                self.pos += 1;
                while (self.pos < self.buf.len and self.buf[self.pos] != '\n' and self.buf[self.pos] != '\r') {
                    self.pos += 1;
                }
            } else if (b >= 0 and b <= 31) {
                self.pushMessageHere(.AsciiControl);
                self.pos += 1;
            } else {
                break;
            }
        }
    }

    pub fn chompNumber(self: *Cursor) Token.Tag {
        const initialDigit = self.buf[self.pos];
        self.pos += 1;

        var tok = Token.Tag.Int;
        if (initialDigit == '0') {
            while (true) {
                const c = self.peek() orelse 0;
                switch (c) {
                    'x', 'X' => {
                        if (c == 'X') {
                            self.pushMessageHere(.UppercaseBase);
                        }
                        self.pos += 1;
                        self.chompIntegerBase16() catch {
                            tok = .MalformedNumberNoDigits;
                        };
                        tok = self.chompNumberSuffix(tok);
                        break;
                    },
                    'o', 'O' => {
                        if (c == 'O') {
                            self.pushMessageHere(.UppercaseBase);
                        }
                        self.pos += 1;
                        self.chompIntegerBase8() catch {
                            tok = .MalformedNumberNoDigits;
                        };
                        tok = self.chompNumberSuffix(tok);
                        break;
                    },
                    'b', 'B' => {
                        if (c == 'B') {
                            self.pushMessageHere(.UppercaseBase);
                        }
                        self.pos += 1;
                        self.chompIntegerBase2() catch {
                            tok = .MalformedNumberNoDigits;
                        };
                        tok = self.chompNumberSuffix(tok);
                        break;
                    },
                    '0'...'9' => {
                        self.pushMessageHere(.LeadingZero);
                        tok = self.chompNumberBase10();
                        tok = self.chompNumberSuffix(tok);
                        break;
                    },
                    '_' => {
                        self.pos += 1;
                        continue;
                    },
                    '.' => {
                        self.pos -= 1; // Go back to the initial 0
                        tok = self.chompNumberBase10();
                        tok = self.chompNumberSuffix(tok);
                        break;
                    },
                    else => {
                        tok = self.chompNumberSuffix(tok);
                        break;
                    },
                }
            }
        } else {
            tok = self.chompNumberBase10();
            tok = self.chompNumberSuffix(tok);
        }
        return tok;
    }

    /// Chomps an exponent including sign and digits, if one if found.
    /// Returns true if an exponent was chomped.
    /// Will error if the exponent has no digits.
    pub fn chompExponent(self: *Cursor) error{EmptyExponent}!bool {
        if (self.peek() orelse 0 == 'e' or self.peek() orelse 0 == 'E') {
            self.pos += 1;
            // Optional sign
            if (self.peek() orelse 0 == '+' or self.peek() orelse 0 == '-') {
                self.pos += 1;
            }
            self.chompIntegerBase10() catch {
                return error.EmptyExponent;
            };
            return true;
        }
        return false;
    }

    /// Chomp what's expected to be the suffix of a number.
    /// Returns either the original token hypothesis, or a malformed token tag.
    pub fn chompNumberSuffix(self: *Cursor, hypothesis: Token.Tag) Token.Tag {
        if (self.peek()) |c| {
            const is_ident_char = (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or (c >= '0' and c <= '9') or c == '_' or c == '$' or c >= 0x80;
            if (!is_ident_char) {
                return hypothesis;
            }
        } else {
            return hypothesis;
        }
        const start = self.pos;
        if (!self.chompIdentGeneral()) {
            return hypothesis.updateIfNotMalformed(.MalformedNumberUnicodeSuffix);
        }
        const suffix = self.buf[start..self.pos];
        if (Token.valid_number_suffixes.get(suffix) == null) {
            return hypothesis.updateIfNotMalformed(.MalformedNumberBadSuffix);
        } else {
            return hypothesis;
        }
    }

    /// Chomp a number in base 10. The number can be an int or float.
    /// Returns the tag of the number type.
    /// Will return a malformed node if the exponent is malformed.
    /// Before calling this method, a valid leading digit must have been parsed.
    pub fn chompNumberBase10(self: *Cursor) Token.Tag {
        var token_type: Token.Tag = .Int;
        self.chompIntegerBase10() catch {}; // This is not an issue, have leading digit.
        if (self.peek() orelse 0 == '.' and (self.isPeekedCharInRange(1, '0', '9') or self.peekAt(1) == 'e' or self.peekAt(1) == 'E')) {
            self.pos += 1;
            self.chompIntegerBase10() catch {}; // This is not an issue, guaranteed to have digits before the decimal point.
            token_type = .Float;
        }
        const has_exponent = self.chompExponent() catch {
            return .MalformedNumberNoExponentDigits;
        };
        if (has_exponent) {
            token_type = .Float;
        }
        return token_type;
    }

    /// Chomp the digits of an integer in base 10.
    /// Will error if the integer has no digits.
    pub fn chompIntegerBase10(self: *Cursor) error{EmptyInteger}!void {
        var contains_digits = false;
        while (self.peek()) |c| {
            if (c >= '0' and c <= '9') {
                contains_digits = true;
                self.pos += 1;
            } else if (c == '_') {
                self.pos += 1;
            } else {
                break;
            }
        }
        if (!contains_digits) {
            return error.EmptyInteger;
        }
    }

    /// Chomp the digits of an integer in base 16.
    /// Will error if the integer has no digits.
    pub fn chompIntegerBase16(self: *Cursor) error{EmptyInteger}!void {
        var contains_digits = false;
        while (self.peek()) |c| {
            if ((c >= '0' and c <= '9') or (c >= 'a' and c <= 'f') or (c >= 'A' and c <= 'F')) {
                contains_digits = true;
                self.pos += 1;
            } else if (c == '_') {
                self.pos += 1;
            } else {
                break;
            }
        }
        if (!contains_digits) {
            return error.EmptyInteger;
        }
    }

    /// Chomp the digits of an integer in base 8.
    /// Will error if the integer has no digits.
    pub fn chompIntegerBase8(self: *Cursor) error{EmptyInteger}!void {
        var contains_digits = false;
        while (self.peek()) |c| {
            if (c >= '0' and c <= '7') {
                contains_digits = true;
                self.pos += 1;
            } else if (c == '_') {
                self.pos += 1;
            } else {
                break;
            }
        }
        if (!contains_digits) {
            return error.EmptyInteger;
        }
    }

    /// Chomp the digits of an integer in base 2.
    /// Will error if the integer has no digits.
    pub fn chompIntegerBase2(self: *Cursor) error{EmptyInteger}!void {
        var contains_digits = false;
        while (self.peek()) |c| {
            if (c == '0' or c == '1') {
                contains_digits = true;
                self.pos += 1;
            } else if (c == '_') {
                self.pos += 1;
            } else {
                break;
            }
        }
        if (!contains_digits) {
            return error.EmptyInteger;
        }
    }

    /// Chomps an identifier starting with a lowercase letter.
    /// Also checks if the resulting identifier is a keyword.
    /// Returns the token type - LowerIdent or Kw*
    pub fn chompIdentLower(self: *Cursor) Token.Tag {
        const start = self.pos;
        if (!self.chompIdentGeneral()) {
            return .MalformedUnicodeIdent;
        }
        const ident = self.buf[start..self.pos];
        const kw = Token.keywords.get(ident);
        return kw orelse .LowerIdent;
    }

    /// Chomps a general identifier - either upper or lower case.
    /// Doesn't check if the identifier is a keyword, since we assume the caller already
    /// determined that was impossible (e.g. because the first character was uppercase),
    /// or otherwise not relevant.
    ///
    /// Returns whether the chomped identifier was valid - i.e. didn't contain any non-ascii characters.
    pub fn chompIdentGeneral(self: *Cursor) bool {
        var valid = true;
        const start_pos = self.pos;
        while (self.pos < self.buf.len) {
            const c = self.buf[self.pos];
            if ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or (c >= '0' and c <= '9') or c == '_' or c == '!' or c == '$') {
                // Allow $ as a valid identifier character
                if (c == '$' and self.pos > start_pos) {
                    // But warn if it's not at the start (pos > start_pos means we've moved)
                    // Use pushMessage to specify the exact location of the $ character
                    self.pushMessage(.DollarInMiddleOfIdentifier, self.pos, self.pos + 1);
                }
                self.pos += 1;
            } else if (c >= 0x80) {
                valid = false;
                self.pos += 1;
            } else {
                break;
            }
        }
        return valid;
    }

    pub fn chompInteger(self: *Cursor) void {
        while (self.pos < self.buf.len) {
            const c = self.buf[self.pos];
            if (c >= '0' and c <= '9') {
                self.pos += 1;
            } else {
                break;
            }
        }
    }

    pub fn chompEscapeSequence(self: *Cursor) error{ InvalidUnicodeEscapeSequence, InvalidEscapeSequence }!void {
        return self.chompEscapeSequenceWithQuote(null);
    }

    pub fn chompEscapeSequenceWithQuote(self: *Cursor, quote_char: ?u8) error{ InvalidUnicodeEscapeSequence, InvalidEscapeSequence }!void {
        // Store the start position of the escape sequence (before the backslash)
        const escape_start = if (self.pos > 0) self.pos - 1 else self.pos;

        switch (self.peek() orelse 0) {
            '\\', '"', '\'', 'n', 'r', 't', '$' => {
                self.pos += 1;
            },
            'u' => {
                self.pos += 1;

                if (self.peek() == '(') {
                    self.pos += 1;
                } else {
                    self.pushMessage(.InvalidUnicodeEscapeSequence, escape_start, self.pos);
                    return error.InvalidUnicodeEscapeSequence;
                }

                const hex_start = self.pos;
                while (true) {
                    if (self.peek() == ')') {
                        if (self.pos == hex_start) {
                            // Empty unicode escape sequence
                            self.pushMessage(.InvalidUnicodeEscapeSequence, escape_start, self.pos + 1);
                            self.pos += 1;
                            return error.InvalidUnicodeEscapeSequence;
                        }
                        self.pos += 1;
                        break;
                    } else if (self.peek() != null) {
                        const next = self.peek() orelse 0;
                        if ((next >= '0' and next <= '9') or
                            (next >= 'a' and next <= 'f') or
                            (next >= 'A' and next <= 'F'))
                        {
                            self.pos += 1;
                        } else {
                            // Invalid hex character - advance to the closing paren if possible
                            // to include the full escape sequence in the error region, but stop
                            // if we encounter the closing quote or newline
                            while (self.pos < self.buf.len) {
                                const next_char = self.peek() orelse 0;
                                if (next_char == ')' or next_char == '\n') {
                                    break;
                                }
                                if (quote_char) |qc| {
                                    if (next_char == qc) {
                                        break;
                                    }
                                }
                                self.pos += 1;
                            }
                            if (self.pos < self.buf.len and self.peek() == ')') {
                                self.pos += 1;
                            }
                            self.pushMessage(.InvalidUnicodeEscapeSequence, escape_start, self.pos);
                            return error.InvalidUnicodeEscapeSequence;
                        }
                    } else {
                        self.pushMessage(.InvalidUnicodeEscapeSequence, escape_start, self.pos);
                        return error.InvalidUnicodeEscapeSequence;
                    }
                }
                const hex_code = self.buf[hex_start .. self.pos - 1];
                const codepoint = std.fmt.parseInt(u21, hex_code, 16) catch {
                    self.pushMessage(.InvalidUnicodeEscapeSequence, escape_start, self.pos);
                    return error.InvalidUnicodeEscapeSequence;
                };

                if (!std.unicode.utf8ValidCodepoint(codepoint)) {
                    self.pushMessage(.InvalidUnicodeEscapeSequence, escape_start, self.pos);
                    return error.InvalidUnicodeEscapeSequence;
                }
            },
            else => {
                // Include the character after the backslash in the error region
                const end_pos = if (self.peek() != null) self.pos + 1 else self.pos;
                self.pushMessage(.InvalidEscapeSequence, escape_start, end_pos);
                return error.InvalidEscapeSequence;
            },
        }
    }

    pub fn chompSingleQuoteLiteral(self: *Cursor) Token.Tag {
        const State = union(enum) {
            Empty,
            Enough,
            TooLong,
            Invalid,
        };

        std.debug.assert(self.peek() == '\'');
        const start = self.pos;

        // Skip the initial quote.
        self.pos += 1;
        var state: State = .Empty;

        while (self.pos < self.buf.len) {
            const c = self.buf[self.pos];

            if (c == '\n') {
                break;
            }

            self.pos += 1;

            switch (state) {
                .Empty => switch (c) {
                    '\'' => {
                        self.pushMessage(.SingleQuoteEmpty, start, self.pos);
                        return .MalformedSingleQuote;
                    },
                    '\\' => {
                        state = .Enough;
                        self.chompEscapeSequenceWithQuote('\'') catch {
                            state = .Invalid;
                        };
                    },
                    else => {
                        self.pos -= 1;
                        if (self.chompUTF8CodepointWithValidation()) |_| {} else {}
                        state = .Enough;
                    },
                },
                .Enough => switch (c) {
                    '\'' => {
                        return .SingleQuote;
                    },
                    else => {
                        state = .TooLong;
                    },
                },
                .TooLong => switch (c) {
                    '\'' => {
                        self.pushMessage(.SingleQuoteTooLong, start, self.pos);
                        return .MalformedSingleQuote;
                    },
                    else => {},
                },
                .Invalid => switch (c) {
                    '\'' => {
                        return .MalformedSingleQuote;
                    },
                    else => {},
                },
            }
        }

        self.pushMessage(.SingleQuoteUnclosed, start, self.pos);
        return .MalformedSingleQuote;
    }

    /// Chomps a UTF-8 codepoint and advances the cursor position.
    /// Reports InvalidUtf8InSource diagnostic if the sequence is invalid or incomplete.
    /// Reports NonPrintableUnicodeInStrLiteral for non-printable characters.
    /// Returns the decoded codepoint if valid, null if invalid.
    fn chompUTF8CodepointWithValidation(self: *Cursor) ?u21 {
        std.debug.assert(self.pos < self.buf.len);
        const c = self.buf[self.pos];

        if (c < 0x80) {
            // Single-byte UTF-8 (ASCII)
            // Allow tab (0x09) in addition to standard printable ASCII
            if (!std.ascii.isPrint(c) and c != '\t') {
                self.pushMessageHere(.NonPrintableUnicodeInStrLiteral);
            }
            self.pos += 1;
            return c;
        }

        const utf8_len = std.unicode.utf8ByteSequenceLength(c) catch {
            self.pushMessageHere(.InvalidUtf8InSource);
            self.pos += 1;
            return null;
        };
        if (self.pos + utf8_len > self.buf.len) {
            // Incomplete UTF-8 sequence at end of input
            self.pushMessageHere(.InvalidUtf8InSource);
            self.pos += 1;
            return null;
        }
        const utf8_bytes = self.buf[self.pos .. self.pos + utf8_len];
        const codepoint = std.unicode.utf8Decode(utf8_bytes) catch {
            self.pushMessageHere(.InvalidUtf8InSource);
            self.pos += 1;
            return null;
        };

        // Check if the Unicode codepoint is printable
        // Unicode categories: Cc (control), Cn (unassigned), Co (private use), Cs (surrogate)
        if (codepoint < 0x20 or // C0 control characters
            (codepoint >= 0x7F and codepoint <= 0x9F) or // C1 control characters
            (codepoint >= 0xD800 and codepoint <= 0xDFFF) or // Surrogate pairs (invalid in UTF-8)
            codepoint == 0xFFFE or codepoint == 0xFFFF) // Non-characters
        {
            self.pushMessageHere(.NonPrintableUnicodeInStrLiteral);
        }

        // Valid UTF-8 sequence - advance by the full sequence length
        self.pos += utf8_len;
        return codepoint;
    }
};

/// The output of the tokenizer.
pub const TokenOutput = struct {
    tokens: TokenizedBuffer,
    messages: []Diagnostic,
    extra_messages_dropped: usize,
};

const StringKind = enum {
    single_line,
    multi_line,
};

/// The tokenizer that uses a Cursor and produces a TokenizedBuffer.
pub const Tokenizer = struct {
    cursor: Cursor,
    output: TokenizedBuffer,
    string_interpolation_stack: std.array_list.Managed(StringKind),
    env: *CommonEnv,

    /// Creates a new Tokenizer.
    /// Note that the caller must also provide a pre-allocated messages buffer.
    pub fn init(env: *CommonEnv, gpa: std.mem.Allocator, text: []const u8, messages: []Diagnostic) std.mem.Allocator.Error!Tokenizer {
        const cursor = Cursor.init(text, messages);
        // TODO: tune this more. Syntax grab bag is 3:1.
        // Generally, roc code will be less dense than that.
        const output = try TokenizedBuffer.initCapacity(env, gpa, text.len);
        return Tokenizer{
            .cursor = cursor,
            .output = output,
            .string_interpolation_stack = std.array_list.Managed(StringKind).init(gpa),
            .env = env,
        };
    }

    pub fn deinit(self: *Tokenizer, gpa: std.mem.Allocator) void {
        self.output.deinit(gpa);
        self.string_interpolation_stack.deinit();
    }

    pub fn finishAndDeinit(self: *Tokenizer) TokenOutput {
        self.string_interpolation_stack.deinit();
        const actual_message_count = @min(self.cursor.message_count, self.cursor.messages.len);
        return .{
            .tokens = self.output,
            .messages = self.cursor.messages[0..actual_message_count],
            .extra_messages_dropped = self.cursor.message_count - actual_message_count,
        };
    }

    /// Returns the tag of the last token pushed, or null if no tokens have been pushed yet.
    fn lastTokenTag(self: *const Tokenizer) ?Token.Tag {
        const len = self.output.tokens.len;
        if (len == 0) return null;
        return self.output.tokens.items(.tag)[len - 1];
    }

    /// Pushes a token with the given tag, token offset, and extra.
    fn pushTokenNormalHere(self: *Tokenizer, gpa: std.mem.Allocator, tag: Token.Tag, tok_offset: Token.Idx) std.mem.Allocator.Error!void {
        std.debug.assert(!tag.isInterned());
        try self.output.tokens.append(gpa, .{
            .tag = tag,
            .region = base.Region.from_raw_offsets(tok_offset, self.cursor.pos),
            .extra = .{ .none = 0 },
        });
    }

    fn pushTokenInternedHere(
        self: *Tokenizer,
        gpa: std.mem.Allocator,
        tag: Token.Tag,
        tok_offset: Token.Idx,
        text_offset: Token.Idx,
    ) Allocator.Error!void {
        const text = self.cursor.buf[text_offset..self.cursor.pos];
        const id = try self.env.insertIdent(gpa, base.Ident.for_text(text));

        // Use underscore flags for type variable identifiers that benefit from fast checking
        if (tag.hasUnderscoreFlags()) {
            // Calculate underscore flags for identifier tokens
            const starts_with_underscore = text.len > 0 and text[0] == '_';
            const ends_with_underscore = text.len > 1 and text[text.len - 1] == '_';

            try self.output.tokens.append(gpa, .{
                .tag = tag,
                .extra = .{ .ident_with_flags = .{
                    .ident = id,
                    .starts_with_underscore = starts_with_underscore,
                    .ends_with_underscore = ends_with_underscore,
                } },
                .region = base.Region.from_raw_offsets(tok_offset, self.cursor.pos),
            });
        } else {
            std.debug.assert(tag.isInterned());
            try self.output.tokens.append(gpa, .{
                .tag = tag,
                .extra = .{ .interned = id },
                .region = base.Region.from_raw_offsets(tok_offset, self.cursor.pos),
            });
        }
    }

    /// The main tokenize loop. This loops over the whole input buffer, tokenizing as it goes.
    pub fn tokenize(self: *Tokenizer, gpa: std.mem.Allocator) std.mem.Allocator.Error!void {
        const trace = tracy.trace(@src());
        defer trace.end();

        var sawWhitespace: bool = true;
        while (self.cursor.pos < self.cursor.buf.len) {
            const start = self.cursor.pos;
            const sp = sawWhitespace;
            sawWhitespace = false;
            const b = self.cursor.buf[self.cursor.pos];
            switch (b) {
                // Whitespace & control characters
                0...32, '#' => {
                    self.cursor.chompTrivia();
                    sawWhitespace = true;
                },

                // Dot (.)
                '.' => {
                    const next = self.cursor.peekAt(1);
                    if (next) |n| {
                        if (n == '.') {
                            if (self.cursor.peekAt(2) == '.') {
                                self.cursor.pos += 3;
                                try self.pushTokenNormalHere(gpa, .TripleDot, start);
                            } else {
                                self.cursor.pos += 2;
                                try self.pushTokenNormalHere(gpa, .DoubleDot, start);
                            }
                        } else if (n >= '0' and n <= '9') {
                            self.cursor.pos += 1;
                            self.cursor.chompInteger();
                            try self.pushTokenNormalHere(gpa, if (sp) .DotInt else .NoSpaceDotInt, start);
                        } else if (n >= 'a' and n <= 'z') {
                            var tag: Token.Tag = if (sp) .DotLowerIdent else .NoSpaceDotLowerIdent;
                            self.cursor.pos += 1;
                            const text_start = self.cursor.pos;
                            if (!self.cursor.chompIdentGeneral()) {
                                tag = .MalformedDotUnicodeIdent;
                            }
                            try self.pushTokenInternedHere(gpa, tag, start, text_start);
                        } else if (n >= 'A' and n <= 'Z') {
                            var tag: Token.Tag = if (sp) .DotUpperIdent else .NoSpaceDotUpperIdent;
                            self.cursor.pos += 1;
                            const text_start = self.cursor.pos;
                            if (!self.cursor.chompIdentGeneral()) {
                                tag = .MalformedDotUnicodeIdent;
                            }
                            try self.pushTokenInternedHere(gpa, tag, start, text_start);
                        } else if (n >= 0x80 and n <= 0xff) {
                            self.cursor.pos += 1;
                            const text_start = self.cursor.pos;
                            const ok = self.cursor.chompIdentGeneral();
                            if (!ok) {
                                // malformed unicode ident remains malformed
                            }
                            try self.pushTokenInternedHere(gpa, .MalformedDotUnicodeIdent, start, text_start);
                        } else if (n == open_curly) {
                            self.cursor.pos += 1;
                            try self.pushTokenNormalHere(gpa, .Dot, start);
                        } else if (n == '*') {
                            self.cursor.pos += 2;
                            try self.pushTokenNormalHere(gpa, .DotStar, start);
                        } else {
                            self.cursor.pos += 1;
                            try self.pushTokenNormalHere(gpa, .Dot, start);
                        }
                    } else {
                        self.cursor.pos += 1;
                        try self.pushTokenNormalHere(gpa, .Dot, start);
                    }
                },

                // Minus (-)
                '-' => {
                    const next = self.cursor.peekAt(1);
                    if (next) |n| {
                        if (n == '>') {
                            self.cursor.pos += 2;
                            try self.pushTokenNormalHere(gpa, .OpArrow, start);
                        } else if (n == ' ' or n == '\t' or n == '\n' or n == '\r' or n == '#') {
                            self.cursor.pos += 1;
                            try self.pushTokenNormalHere(gpa, .OpBinaryMinus, start);
                        } else if (n >= '0' and n <= '9') {
                            // When there's no whitespace before the minus (sp == false) and
                            // the previous token can end an expression, this is binary minus
                            // followed by a number (e.g., x-1), not a negative number literal.
                            // When there is whitespace before minus (sp == true), treat -N as
                            // a negative literal (e.g., `if True -10 else x`).
                            const prev_can_end_expr = if (self.lastTokenTag()) |prev| prev.canEndExpression() else false;
                            if (!sp and prev_can_end_expr) {
                                self.cursor.pos += 1;
                                try self.pushTokenNormalHere(gpa, .OpBinaryMinus, start);
                            } else {
                                self.cursor.pos += 1;
                                const tag = self.cursor.chompNumber();
                                try self.pushTokenNormalHere(gpa, tag, start);
                            }
                        } else {
                            self.cursor.pos += 1;
                            // Look at what follows the minus to determine if it's unary
                            const tokenType: Token.Tag = if (canFollowUnaryMinus(n)) .OpUnaryMinus else .OpBinaryMinus;
                            try self.pushTokenNormalHere(gpa, tokenType, start);
                        }
                    } else {
                        self.cursor.pos += 1;
                        // No next character - default to unary
                        try self.pushTokenNormalHere(gpa, .OpUnaryMinus, start);
                    }
                },

                // Exclamation (!)
                '!' => {
                    if (self.cursor.peekAt(1) == '=') {
                        self.cursor.pos += 2;
                        try self.pushTokenNormalHere(gpa, .OpNotEquals, start);
                    } else {
                        self.cursor.pos += 1;
                        try self.pushTokenNormalHere(gpa, .OpBang, start);
                    }
                },

                // Ampersand (&)
                '&' => {
                    self.cursor.pos += 1;
                    try self.pushTokenNormalHere(gpa, .OpAmpersand, start);
                },

                // Comma (,)
                ',' => {
                    self.cursor.pos += 1;
                    try self.pushTokenNormalHere(gpa, .Comma, start);
                },

                // Question mark (?)
                '?' => {
                    if (self.cursor.peekAt(1) == '?') {
                        self.cursor.pos += 2;
                        try self.pushTokenNormalHere(gpa, .OpDoubleQuestion, start);
                    } else {
                        self.cursor.pos += 1;
                        try self.pushTokenNormalHere(gpa, if (sp) .OpQuestion else .NoSpaceOpQuestion, start);
                    }
                },

                // Pipe (|)
                '|' => {
                    if (self.cursor.peekAt(1) == '>') {
                        self.cursor.pos += 2;
                        try self.pushTokenNormalHere(gpa, .OpPizza, start);
                    } else {
                        self.cursor.pos += 1;
                        try self.pushTokenNormalHere(gpa, .OpBar, start);
                    }
                },

                // Plus (+)
                '+' => {
                    self.cursor.pos += 1;
                    try self.pushTokenNormalHere(gpa, .OpPlus, start);
                },

                // Star (*)
                '*' => {
                    self.cursor.pos += 1;
                    try self.pushTokenNormalHere(gpa, .OpStar, start);
                },

                // Slash (/)
                '/' => {
                    if (self.cursor.peekAt(1) == '/') {
                        self.cursor.pos += 2;
                        try self.pushTokenNormalHere(gpa, .OpDoubleSlash, start);
                    } else {
                        self.cursor.pos += 1;
                        try self.pushTokenNormalHere(gpa, .OpSlash, start);
                    }
                },

                // Backslash (\)
                '\\' => {
                    if (self.cursor.peekAt(1) == '\\') {
                        try self.tokenizeMultilineStringLiteral(gpa);
                    } else {
                        self.cursor.pos += 1;
                        try self.pushTokenNormalHere(gpa, .OpBackslash, start);
                    }
                },

                // Percent (%)
                '%' => {
                    self.cursor.pos += 1;
                    try self.pushTokenNormalHere(gpa, .OpPercent, start);
                },

                // Caret (^)
                '^' => {
                    self.cursor.pos += 1;
                    try self.pushTokenNormalHere(gpa, .OpCaret, start);
                },

                // Greater-than (>)
                '>' => {
                    if (self.cursor.peekAt(1) == '=') {
                        self.cursor.pos += 2;
                        try self.pushTokenNormalHere(gpa, .OpGreaterThanOrEq, start);
                    } else {
                        self.cursor.pos += 1;
                        try self.pushTokenNormalHere(gpa, .OpGreaterThan, start);
                    }
                },

                // Less-than (<)
                '<' => {
                    if (self.cursor.peekAt(1) == '=') {
                        self.cursor.pos += 2;
                        try self.pushTokenNormalHere(gpa, .OpLessThanOrEq, start);
                    } else if (self.cursor.peekAt(1) == '-') {
                        self.cursor.pos += 2;
                        try self.pushTokenNormalHere(gpa, .OpBackArrow, start);
                    } else {
                        self.cursor.pos += 1;
                        try self.pushTokenNormalHere(gpa, .OpLessThan, start);
                    }
                },

                // Equals (=)
                '=' => {
                    if (self.cursor.peekAt(1) == '=') {
                        self.cursor.pos += 2;
                        try self.pushTokenNormalHere(gpa, .OpEquals, start);
                    } else if (self.cursor.peekAt(1) == '>') {
                        self.cursor.pos += 2;
                        try self.pushTokenNormalHere(gpa, .OpFatArrow, start);
                    } else {
                        self.cursor.pos += 1;
                        try self.pushTokenNormalHere(gpa, .OpAssign, start);
                    }
                },

                // Colon (:)
                ':' => {
                    if (self.cursor.peekAt(1) == '=') {
                        self.cursor.pos += 2;
                        try self.pushTokenNormalHere(gpa, .OpColonEqual, start);
                    } else if (self.cursor.peekAt(1) == ':') {
                        self.cursor.pos += 2;
                        try self.pushTokenNormalHere(gpa, .OpDoubleColon, start);
                    } else {
                        self.cursor.pos += 1;
                        try self.pushTokenNormalHere(gpa, .OpColon, start);
                    }
                },

                '(' => {
                    self.cursor.pos += 1;
                    try self.pushTokenNormalHere(gpa, if (sp) .OpenRound else .NoSpaceOpenRound, start);
                },
                '[' => {
                    self.cursor.pos += 1;
                    try self.pushTokenNormalHere(gpa, .OpenSquare, start);
                },
                open_curly => {
                    self.cursor.pos += 1;
                    try self.pushTokenNormalHere(gpa, .OpenCurly, start);
                },

                ')' => {
                    self.cursor.pos += 1;
                    try self.pushTokenNormalHere(gpa, .CloseRound, start);
                },
                ']' => {
                    self.cursor.pos += 1;
                    try self.pushTokenNormalHere(gpa, .CloseSquare, start);
                },
                close_curly => {
                    self.cursor.pos += 1;
                    if (self.string_interpolation_stack.pop()) |last| {
                        try self.pushTokenNormalHere(gpa, .CloseStringInterpolation, start);
                        // For string interpolations, we don't have the original opening quote position
                        // so we use the current position as a fallback
                        try self.tokenizeStringLikeLiteralBody(gpa, last, self.cursor.pos);
                    } else {
                        try self.pushTokenNormalHere(gpa, .CloseCurly, start);
                    }
                },

                '_' => {
                    const next = self.cursor.peekAt(1);
                    if (next) |n| {
                        if ((n >= 'a' and n <= 'z') or (n >= 'A' and n <= 'Z') or (n >= '0' and n <= '9')) {
                            var tok: Token.Tag = .NamedUnderscore;
                            self.cursor.pos += 2;
                            if (!self.cursor.chompIdentGeneral()) {
                                tok = .MalformedNamedUnderscoreUnicode;
                            }
                            try self.pushTokenInternedHere(gpa, tok, start, start);
                        } else {
                            self.cursor.pos += 1;
                            try self.pushTokenNormalHere(gpa, .Underscore, start);
                        }
                    } else {
                        self.cursor.pos += 1;
                        try self.pushTokenNormalHere(gpa, .Underscore, start);
                    }
                },

                '@' => {
                    var tok: Token.Tag = .OpaqueName;
                    const next = self.cursor.peekAt(1);
                    if (next) |n| {
                        if ((n >= 'a' and n <= 'z') or (n >= 'A' and n <= 'Z') or (n >= '0' and n <= '9') or n == '_' or n >= 0x80) {
                            self.cursor.pos += 1;
                            if (!self.cursor.chompIdentGeneral()) {
                                tok = .MalformedOpaqueNameUnicode;
                            }
                        } else {
                            tok = .MalformedOpaqueNameWithoutName;
                            self.cursor.pos += 1;
                        }
                    } else {
                        tok = .MalformedOpaqueNameWithoutName;
                        self.cursor.pos += 1;
                    }
                    if (tok.isInterned()) {
                        try self.pushTokenInternedHere(gpa, tok, start, start);
                    } else {
                        try self.pushTokenNormalHere(gpa, tok, start);
                    }
                },

                '$' => {
                    const next = self.cursor.peekAt(1);
                    if (next) |n| {
                        if (n >= 'a' and n <= 'z') {
                            // Dollar sign followed by lowercase letter - reusable identifier
                            var tag: Token.Tag = .LowerIdent;
                            self.cursor.pos += 1;
                            if (!self.cursor.chompIdentGeneral()) {
                                tag = .MalformedUnicodeIdent;
                            }
                            try self.pushTokenInternedHere(gpa, tag, start, start);
                        } else if (n >= 'A' and n <= 'Z') {
                            // Dollar sign followed by uppercase letter - reusable identifier
                            var tag: Token.Tag = .UpperIdent;
                            self.cursor.pos += 1;
                            if (!self.cursor.chompIdentGeneral()) {
                                tag = .MalformedUnicodeIdent;
                            }
                            try self.pushTokenInternedHere(gpa, tag, start, start);
                        } else {
                            // Dollar sign not followed by a letter - invalid
                            self.cursor.pos += 1;
                            try self.pushTokenNormalHere(gpa, .MalformedUnknownToken, start);
                        }
                    } else {
                        // Dollar sign at end of file
                        self.cursor.pos += 1;
                        try self.pushTokenNormalHere(gpa, .MalformedUnknownToken, start);
                    }
                },

                // Numbers starting with 0-9
                '0'...'9' => {
                    const tag = self.cursor.chompNumber();
                    try self.pushTokenNormalHere(gpa, tag, start);
                },

                // Lowercase identifiers
                'a'...'z' => {
                    const tag = self.cursor.chompIdentLower();
                    if (tag == .LowerIdent or tag == .MalformedUnicodeIdent) {
                        try self.pushTokenInternedHere(gpa, tag, start, start);
                    } else {
                        try self.pushTokenNormalHere(gpa, tag, start);
                    }
                },

                // Uppercase identifiers
                'A'...'Z' => {
                    var tag: Token.Tag = .UpperIdent;
                    if (!self.cursor.chompIdentGeneral()) {
                        tag = .MalformedUnicodeIdent;
                    }
                    try self.pushTokenInternedHere(gpa, tag, start, start);
                },

                '\'' => {
                    const tag = self.cursor.chompSingleQuoteLiteral();
                    try self.pushTokenNormalHere(gpa, tag, start);
                },

                '"' => {
                    // Note this may return StringStart/StringPart instead of String,
                    // in the case of a string interpolation.
                    try self.tokenizeStringLikeLiteral(gpa);
                },

                // first byte of a UTF-8 sequence
                0x80...0xff => {
                    const valid = self.cursor.chompIdentGeneral();
                    if (comptime @import("builtin").mode == .Debug) {
                        std.debug.assert(!valid);
                    } else if (valid) {
                        unreachable;
                    }
                    try self.pushTokenInternedHere(gpa, .MalformedUnicodeIdent, start, start);
                },

                // TODO: emit a MalformedOpToken for invalid combinations of operator-like characters

                // Fallback for any unknown token.
                else => {
                    self.cursor.pos += 1;
                    try self.pushTokenNormalHere(gpa, .MalformedUnknownToken, start);
                },
            }
        }

        try self.pushTokenNormalHere(gpa, .EndOfFile, self.cursor.pos);
    }

    /// Determines if a character can follow a unary minus (i.e., can start an expression)
    fn canFollowUnaryMinus(c: u8) bool {
        return switch (c) {
            // Identifiers
            'a'...'z', 'A'...'Z', '_' => true,
            // Parentheses for grouped expressions
            '(' => true,
            // Unicode characters that might start identifiers
            0x80...0xFF => true,
            // Everything else suggests binary minus
            else => false,
        };
    }

    pub fn tokenizeStringLikeLiteral(self: *Tokenizer, gpa: std.mem.Allocator) std.mem.Allocator.Error!void {
        const start = self.cursor.pos;
        std.debug.assert(self.cursor.peek() == '"');
        self.cursor.pos += 1;
        var kind: StringKind = .single_line;
        if (self.cursor.peek() == '"' and self.cursor.peekAt(1) == '"') {
            self.cursor.pos += 2;
            kind = .multi_line;
            try self.pushTokenNormalHere(gpa, .MultilineStringStart, start);
        } else {
            try self.pushTokenNormalHere(gpa, .StringStart, start);
        }
        try self.tokenizeStringLikeLiteralBody(gpa, kind, start);
    }

    pub fn tokenizeMultilineStringLiteral(self: *Tokenizer, gpa: std.mem.Allocator) std.mem.Allocator.Error!void {
        const start = self.cursor.pos;
        std.debug.assert(self.cursor.peek() == '\\' and self.cursor.peekAt(1) == '\\');
        self.cursor.pos += 2;
        try self.pushTokenNormalHere(gpa, .MultilineStringStart, start);
        try self.tokenizeStringLikeLiteralBody(gpa, .multi_line, start);
    }

    // Moving curly chars to constants because some editors hate them inline.
    const open_curly = '{';
    const close_curly = '}';

    pub fn tokenizeStringLikeLiteralBody(self: *Tokenizer, gpa: std.mem.Allocator, kind: StringKind, opening_quote_pos: usize) std.mem.Allocator.Error!void {
        const start = self.cursor.pos;
        var string_part_tag: Token.Tag = .StringPart;
        while (self.cursor.pos < self.cursor.buf.len) {
            const c = self.cursor.buf[self.cursor.pos];
            if (c == '$' and self.cursor.peekAt(1) == open_curly) {
                try self.pushTokenNormalHere(gpa, string_part_tag, start);
                const dollar_start = self.cursor.pos;
                self.cursor.pos += 2;
                try self.pushTokenNormalHere(gpa, .OpenStringInterpolation, dollar_start);
                try self.string_interpolation_stack.append(kind);
                return;
            } else if (c == '\n') {
                try self.pushTokenNormalHere(gpa, string_part_tag, start);
                if (kind == .single_line) {
                    // Include the opening quote in the error region
                    self.cursor.pushMessage(.UnclosedString, @intCast(opening_quote_pos), @intCast(self.cursor.pos));
                    try self.pushTokenNormalHere(gpa, .StringEnd, self.cursor.pos);
                }
                return;
            } else if (kind == .single_line and c == '"') {
                try self.pushTokenNormalHere(gpa, string_part_tag, start);
                const string_part_end = self.cursor.pos;
                self.cursor.pos += 1;
                try self.pushTokenNormalHere(gpa, .StringEnd, string_part_end);
                return;
            } else {
                // Handle UTF-8 sequences with printable character validation
                if (self.cursor.chompUTF8CodepointWithValidation()) |_| {} else {}

                const escape = c == '\\';
                if (escape) {
                    self.cursor.chompEscapeSequenceWithQuote('"') catch {
                        string_part_tag = .MalformedStringPart;
                    };
                }
            }
        }
        if (kind == .single_line) {
            // Include the opening quote in the error region
            self.cursor.pushMessage(.UnclosedString, @intCast(opening_quote_pos), @intCast(self.cursor.pos));
        }
        try self.pushTokenNormalHere(gpa, string_part_tag, start);
    }
};

fn testTokenization(gpa: std.mem.Allocator, input: []const u8, expected: []const Token.Tag) anyerror!void {
    var messages: [10]Diagnostic = undefined;

    var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
    defer env.deinit(gpa);

    var tokenizer = try Tokenizer.init(&env, gpa, input, &messages);
    defer tokenizer.deinit(gpa);

    try tokenizer.tokenize(gpa);
    const tokenizedBuffer = tokenizer.output;
    const tokens = tokenizedBuffer.tokens.items(.tag);

    try std.testing.expectEqual(tokens[tokens.len - 1], Token.Tag.EndOfFile);
    try std.testing.expectEqualSlices(Token.Tag, expected[0..expected.len], tokens[0 .. tokens.len - 1]);

    try checkTokenizerInvariants(gpa, input, false);
}

/// Assert the invariants of the tokenizer are held.
pub fn checkTokenizerInvariants(gpa: std.mem.Allocator, input: []const u8, debug: bool) std.mem.Allocator.Error!void {
    var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
    defer env.deinit(gpa);

    // Initial tokenization.
    var messages: [32]Diagnostic = undefined;
    var tokenizer = try Tokenizer.init(&env, gpa, input, &messages);
    try tokenizer.tokenize(gpa);
    var output = tokenizer.finishAndDeinit();
    defer output.tokens.deinit(gpa);

    // Debug output (excluded from coverage via --exclude-line=std.debug.print)
    if (debug) {
        std.debug.print("Original:\n==========\n{s}\n==========\n\n", .{input});
    }

    if (debug) {
        std.debug.print("Before:\n", .{});
        for (0..output.tokens.tokens.len) |token_index| {
            const token = output.tokens.tokens.get(token_index);
            std.debug.print("\t{any}\n", .{token});
        }
        std.debug.print("\n\n", .{});
    }

    // TODO: apply errors from messages to buffer below.
    // For now, just skip on tokenizer finding a failure.
    if (output.messages.len != 0) {
        return;
    }

    var buf2 = rebuildBufferForTesting(input, &output.tokens, gpa) catch |err| switch (err) {
        error.Unsupported => return,
        error.OutOfMemory => std.debug.panic("OOM", .{}),
    };
    defer buf2.deinit();

    if (debug) {
        std.debug.print("Intermediate:\n==========\n{s}\n==========\n\n", .{buf2.items});
    }

    // Second tokenization.
    tokenizer = try Tokenizer.init(&env, gpa, buf2.items, &messages);
    try tokenizer.tokenize(gpa);
    var output2 = tokenizer.finishAndDeinit();
    defer output2.tokens.deinit(gpa);

    if (debug) {
        std.debug.print("After:\n", .{});
        for (0..output2.tokens.tokens.len) |token_index| {
            const token = output2.tokens.tokens.get(token_index);
            std.debug.print("\t{any}\n", .{token});
        }
        std.debug.print("\n\n", .{});
    }
    // Assert same.
    var same = output.tokens.tokens.len == output2.tokens.tokens.len;
    for (0..output.tokens.tokens.len) |token_index| {
        if (token_index >= output2.tokens.tokens.len) {
            same = false;
            break;
        }
        const token = output.tokens.tokens.get(token_index);
        const token2 = output2.tokens.tokens.get(token_index);
        const region1 = output.tokens.resolve(token_index);
        const region2 = output2.tokens.resolve(token_index);
        const length1 = region1.end.offset - region1.start.offset;
        const length2 = region2.end.offset - region2.start.offset;
        same = same and (token.tag == token2.tag);
        same = same and (length1 == length2);
    }

    // Diagnostic output when tokens don't match (excluded from coverage via --exclude-line)
    if (!same) {
        var prefix_len: usize = 0;
        var suffix_len: usize = 0;
        while (prefix_len < output.tokens.tokens.len and prefix_len < output2.tokens.tokens.len) : (prefix_len += 1) {
            const token = output.tokens.tokens.get(prefix_len);
            const token2 = output2.tokens.tokens.get(prefix_len);
            const region1 = output.tokens.resolve(prefix_len);
            const region2 = output2.tokens.resolve(prefix_len);

            if (token.tag != token2.tag or region1.start.offset != region2.start.offset) {
                break;
            }
        }

        while (suffix_len < output.tokens.tokens.len - prefix_len and suffix_len < output2.tokens.tokens.len - prefix_len) : (suffix_len += 1) {
            const token = output.tokens.tokens.get(output.tokens.tokens.len - suffix_len - 1);
            const token2 = output2.tokens.tokens.get(output2.tokens.tokens.len - suffix_len - 1);
            const region1 = output.tokens.resolve(output.tokens.tokens.len - suffix_len - 1);
            const region2 = output2.tokens.resolve(output2.tokens.tokens.len - suffix_len - 1);

            if (token.tag != token2.tag or region1.start.offset != region2.start.offset) {
                break;
            }
        }

        std.debug.print("...\n", .{});
        for (prefix_len..output.tokens.tokens.len - suffix_len) |token_index| {
            const region = output.tokens.resolve(token_index);
            const token = output.tokens.tokens.get(token_index);
            std.debug.print("\x1b[31m\t- {any}\x1b[0m: {s}\n", .{ token, input[region.start.offset..region.end.offset] });
        }
        for (prefix_len..output2.tokens.tokens.len - suffix_len) |token_index| {
            const region = output2.tokens.resolve(token_index);
            const token = output2.tokens.tokens.get(token_index);
            std.debug.print("\x1b[32m\t+ {any}\x1b[0m: {s}\n", .{ token, buf2.items[region.start.offset..region.end.offset] });
        }
        std.debug.print("...\n", .{});

        std.debug.assert(same);
    }
}

fn rebuildBufferForTesting(buf: []const u8, tokens: *TokenizedBuffer, alloc: std.mem.Allocator) (Allocator.Error || error{Unsupported})!std.array_list.Managed(u8) {
    // Create an arraylist to store the new buffer.
    var buf2 = try std.array_list.Managed(u8).initCapacity(alloc, buf.len);
    errdefer buf2.deinit();

    // Dump back to buffer.
    // Here we are just printing in the simplest way possible.
    var last_end: usize = 0;
    var prev_token_tag: Token.Tag = .EndOfFile; // placeholder
    for (0..tokens.tokens.len) |token_index| {
        const token = tokens.tokens.get(token_index);
        // EndOfFile is special, handle it early.
        // Unlike other tokens it does not store a correct offset and length
        // EndOfFile consumes the entire file.

        // Copy over limited whitespace.
        // TODO: Long term, we should switch to dummy whitespace, but currently, Roc still has WSS.
        const region = tokens.resolve(token_index);
        for (last_end..region.start.offset) |i| {
            // Leave tabs and newlines alone, they are special to roc.
            // Replace everything else with spaces.
            if (buf[i] != '\t' and buf[i] != '\r' and buf[i] != '\n' and buf[i] != '#') {
                try buf2.append(' ');
            } else {
                try buf2.append(buf[i]);
            }
        }
        if (token.tag == .EndOfFile) {
            break;
        }
        std.debug.assert(region.end.offset >= region.start.offset);
        std.debug.assert(region.end.offset <= buf.len);
        std.debug.assert(region.start.offset >= last_end);
        last_end = region.end.offset;
        const length = region.end.offset - region.start.offset;
        switch (token.tag) {
            .EndOfFile => unreachable,

            .Float => {
                if (buf[region.start.offset] == '-') {
                    try buf2.append('-');
                    try buf2.append('0');
                    try buf2.append('.');
                    for (3..length) |_| {
                        try buf2.append('1');
                    }
                } else {
                    try buf2.append('0');
                    try buf2.append('.');
                    for (2..length) |_| {
                        try buf2.append('1');
                    }
                }
            },
            .SingleQuote => {
                if (length == 3) {
                    try buf2.append('\'');
                    try buf2.append('A');
                    try buf2.append('\'');
                } else if (length == 4) {
                    if (buf[region.start.offset + 1] == '\\') {
                        try buf2.append('\'');
                        try buf2.append('\\');
                        try buf2.append('\\');
                        try buf2.append('\'');
                    } else {
                        try buf2.append('\'');
                        // Å
                        try buf2.append(0xC3);
                        try buf2.append(0x85);
                        try buf2.append('\'');
                    }
                } else if (length == 5) {
                    try buf2.append('\'');
                    // Ἀ
                    try buf2.append(0xE1);
                    try buf2.append(0xBC);
                    try buf2.append(0x88);
                    try buf2.append('\'');
                } else if (length == 6) {
                    try buf2.append('\'');
                    // 𐙝
                    try buf2.append(0xF0);
                    try buf2.append(0x90);
                    try buf2.append(0x99);
                    try buf2.append(0x9D);
                    try buf2.append('\'');
                } else {
                    try buf2.append('\'');
                    try buf2.append('\\');
                    try buf2.append('u');
                    try buf2.append('(');
                    for (6..length) |_| {
                        try buf2.append('0');
                    }
                    try buf2.append(')');
                    try buf2.append('\'');
                }
            },
            .StringStart, .StringEnd => {
                try buf2.append('"');
            },
            .MultilineStringStart => {
                if (length == 3 and buf[region.start.offset] == '"') {
                    try buf2.append('"');
                    try buf2.append('"');
                    try buf2.append('"');
                } else {
                    try buf2.append('\\');
                    try buf2.append('\\');
                }
            },
            .StringPart => {
                for (0..length) |_| {
                    try buf2.append('~');
                }
            },
            .OpenStringInterpolation => {
                std.debug.assert(length == 2);
                try buf2.append('$');
                try buf2.append('{');
            },
            .CloseStringInterpolation => {
                std.debug.assert(length == 1);
                try buf2.append('}');
            },

            .UpperIdent => {
                if (length > 0 and buf[region.start.offset] == '$') {
                    try buf2.append('$');
                    for (1..length) |_| {
                        try buf2.append('Z');
                    }
                } else {
                    try buf2.append('Z');
                    for (1..length) |_| {
                        try buf2.append('z');
                    }
                }
            },
            .LowerIdent => {
                if (length > 0 and buf[region.start.offset] == '$') {
                    try buf2.append('$');
                    for (1..length) |_| {
                        try buf2.append('z');
                    }
                } else {
                    for (0..length) |_| {
                        try buf2.append('z');
                    }
                }
            },
            .Underscore => {
                std.debug.assert(length == 1);
                try buf2.append('_');
            },
            .DotInt => {
                try buf2.append('.');
                for (1..length) |_| {
                    try buf2.append('1');
                }
            },
            .DotLowerIdent => {
                try buf2.append('.');
                for (1..length) |_| {
                    try buf2.append('z');
                }
            },
            .DotUpperIdent => {
                try buf2.append('.');
                try buf2.append('Z');
                for (2..length) |_| {
                    try buf2.append('z');
                }
            },
            .NoSpaceDotInt => {
                try buf2.append('.');
                for (1..length) |_| {
                    try buf2.append('1');
                }
            },
            .NoSpaceDotLowerIdent => {
                try buf2.append('.');
                for (1..length) |_| {
                    try buf2.append('z');
                }
            },
            .NoSpaceDotUpperIdent => {
                try buf2.append('.');
                try buf2.append('Z');
                for (2..length) |_| {
                    try buf2.append('z');
                }
            },
            .NoSpaceOpenRound => {
                try buf2.append('(');
            },

            .NamedUnderscore => {
                try buf2.append('_');
                for (1..length) |_| {
                    try buf2.append('z');
                }
            },
            .OpaqueName => {
                try buf2.append('@');
                for (1..length) |_| {
                    try buf2.append('z');
                }
            },
            .Int => {
                if (buf[region.start.offset] == '-') {
                    try buf2.append('-');
                    if (length >= 4) {
                        try buf2.append('0');
                        try buf2.append('x');
                        for (3..length) |_| {
                            try buf2.append('1');
                        }
                    } else {
                        for (1..length) |_| {
                            try buf2.append('1');
                        }
                    }
                } else if (length >= 3) {
                    // To ensure this value when reprinted tokenizes as an int, add a base if the number is 3 or more characters.
                    try buf2.append('0');
                    try buf2.append('x');
                    for (2..length) |_| {
                        try buf2.append('1');
                    }
                } else {
                    for (0..length) |_| {
                        try buf2.append('1');
                    }
                }
            },

            .OpenRound => {
                std.debug.assert(length == 1);
                try buf2.append('(');
            },
            .CloseRound => {
                std.debug.assert(length == 1);
                try buf2.append(')');
            },
            .OpenSquare => {
                std.debug.assert(length == 1);
                try buf2.append('[');
            },
            .CloseSquare => {
                std.debug.assert(length == 1);
                try buf2.append(']');
            },
            .OpenCurly => {
                std.debug.assert(length == 1);
                try buf2.append('{');
            },
            .CloseCurly => {
                std.debug.assert(length == 1);
                try buf2.append('}');
            },

            .OpPlus => {
                std.debug.assert(length == 1);
                try buf2.append('+');
            },
            .OpStar => {
                std.debug.assert(length == 1);
                try buf2.append('*');
            },
            .OpPizza => {
                std.debug.assert(length == 2);
                try buf2.append('|');
                try buf2.append('>');
            },
            .OpAssign => {
                std.debug.assert(length == 1);
                try buf2.append('=');
            },
            .OpBinaryMinus => {
                std.debug.assert(length == 1);
                try buf2.append('-');
            },
            .OpUnaryMinus => {
                std.debug.assert(length == 1);
                try buf2.append('-');
            },
            .OpNotEquals => {
                std.debug.assert(length == 2);
                try buf2.append('!');
                try buf2.append('=');
            },
            .OpBang => {
                std.debug.assert(length == 1);
                try buf2.append('!');
            },
            .OpAnd => {
                std.debug.assert(length == 3);
                try buf2.append('a');
                try buf2.append('n');
                try buf2.append('d');
            },
            .OpAmpersand => {
                std.debug.assert(length == 1);
                try buf2.append('&');
            },
            .OpQuestion, .NoSpaceOpQuestion => {
                std.debug.assert(length == 1);
                try buf2.append('?');
            },
            .OpDoubleQuestion => {
                std.debug.assert(length == 2);
                try buf2.append('?');
                try buf2.append('?');
            },
            .OpOr => {
                std.debug.assert(length == 2);
                try buf2.append('o');
                try buf2.append('r');
            },
            .OpBar => {
                std.debug.assert(length == 1);
                try buf2.append('|');
            },
            .OpDoubleSlash => {
                std.debug.assert(length == 2);
                try buf2.append('/');
                try buf2.append('/');
            },
            .OpSlash => {
                std.debug.assert(length == 1);
                try buf2.append('/');
            },
            .OpPercent => {
                std.debug.assert(length == 1);
                try buf2.append('%');
            },
            .OpCaret => {
                std.debug.assert(length == 1);
                try buf2.append('^');
            },
            .OpGreaterThanOrEq => {
                std.debug.assert(length == 2);
                try buf2.append('>');
                try buf2.append('=');
            },
            .OpGreaterThan => {
                std.debug.assert(length == 1);
                try buf2.append('>');
            },
            .OpLessThanOrEq => {
                std.debug.assert(length == 2);
                try buf2.append('<');
                try buf2.append('=');
            },
            .OpBackArrow => {
                std.debug.assert(length == 2);
                try buf2.append('<');
                try buf2.append('-');
            },
            .OpLessThan => {
                std.debug.assert(length == 1);
                try buf2.append('<');
            },
            .OpEquals => {
                std.debug.assert(length == 2);
                try buf2.append('=');
                try buf2.append('=');
            },
            .OpColonEqual => {
                std.debug.assert(length == 2);
                try buf2.append(':');
                try buf2.append('=');
            },
            .OpDoubleColon => {
                std.debug.assert(length == 2);
                try buf2.append(':');
                try buf2.append(':');
            },

            .Comma => {
                std.debug.assert(length == 1);
                try buf2.append(',');
            },
            .Dot => {
                std.debug.assert(length == 1);
                try buf2.append('.');
            },
            .DoubleDot => {
                std.debug.assert(length == 2);
                try buf2.append('.');
                try buf2.append('.');
            },
            .TripleDot => {
                std.debug.assert(length == 3);
                try buf2.append('.');
                try buf2.append('.');
                try buf2.append('.');
            },
            .DotStar => {
                std.debug.assert(length == 2);
                try buf2.append('.');
                try buf2.append('*');
            },
            .OpColon => {
                std.debug.assert(length == 1);
                try buf2.append(':');
            },
            .OpArrow => {
                std.debug.assert(length == 2);
                try buf2.append('-');
                try buf2.append('>');
            },
            .OpFatArrow => {
                std.debug.assert(length == 2);
                try buf2.append('=');
                try buf2.append('>');
            },
            .OpBackslash => {
                std.debug.assert(length == 1);
                try buf2.append('\\');
            },

            .KwApp => {
                try buf2.appendSlice("app");
            },
            .KwAs => {
                try buf2.appendSlice("as");
            },
            .KwCrash => {
                try buf2.appendSlice("crash");
            },
            .KwDbg => {
                try buf2.appendSlice("dbg");
            },
            .KwElse => {
                try buf2.appendSlice("else");
            },
            .KwExpect => {
                try buf2.appendSlice("expect");
            },
            .KwExposes => {
                try buf2.appendSlice("exposes");
            },
            .KwExposing => {
                try buf2.appendSlice("exposing");
            },
            .KwFor => {
                try buf2.appendSlice("for");
            },
            .KwGenerates => {
                try buf2.appendSlice("generates");
            },
            .KwHas => {
                try buf2.appendSlice("has");
            },
            .KwHosted => {
                try buf2.appendSlice("hosted");
            },
            .KwIf => {
                try buf2.appendSlice("if");
            },
            .KwImplements => {
                try buf2.appendSlice("implements");
            },
            .KwImport => {
                try buf2.appendSlice("import");
            },
            .KwImports => {
                try buf2.appendSlice("imports");
            },
            .KwIn => {
                try buf2.appendSlice("in");
            },
            .KwInterface => {
                try buf2.appendSlice("interface");
            },
            .KwModule => {
                try buf2.appendSlice("module");
            },
            .KwPackage => {
                try buf2.appendSlice("package");
            },
            .KwPackages => {
                try buf2.appendSlice("packages");
            },
            .KwPlatform => {
                try buf2.appendSlice("platform");
            },
            .KwProvides => {
                try buf2.appendSlice("provides");
            },
            .KwRequires => {
                try buf2.appendSlice("requires");
            },
            .KwReturn => {
                try buf2.appendSlice("return");
            },
            .KwTargets => {
                try buf2.appendSlice("targets");
            },
            .KwVar => {
                try buf2.appendSlice("var");
            },
            .KwMatch => {
                try buf2.appendSlice("match");
            },
            .KwWhere => {
                try buf2.appendSlice("where");
            },
            .KwWhile => {
                try buf2.appendSlice("while");
            },
            .KwWith => {
                try buf2.appendSlice("with");
            },
            .KwBreak => {
                try buf2.appendSlice("break");
            },

            // If the input has malformed tokens, we don't want to assert anything about it (yet)
            .MalformedNumberBadSuffix,
            .MalformedNumberUnicodeSuffix,
            .MalformedNumberNoDigits,
            .MalformedNumberNoExponentDigits,
            .MalformedInvalidUnicodeEscapeSequence,
            .MalformedInvalidEscapeSequence,
            .MalformedUnicodeIdent,
            .MalformedDotUnicodeIdent,
            .MalformedNoSpaceDotUnicodeIdent,
            .MalformedUnknownToken,
            .MalformedNamedUnderscoreUnicode,
            .MalformedOpaqueNameUnicode,
            .MalformedOpaqueNameWithoutName,
            .MalformedSingleQuote,
            .MalformedStringPart,
            => {
                return error.Unsupported;
            },
        }
        prev_token_tag = token.tag;
    }
    return buf2;
}

test "tokenizer" {
    const gpa = std.testing.allocator;
    try testTokenization(gpa, "42", &[_]Token.Tag{.Int});
    try testTokenization(gpa, "3.14", &[_]Token.Tag{.Float});
    try testTokenization(gpa, ".", &[_]Token.Tag{.Dot});
    try testTokenization(gpa, "..", &[_]Token.Tag{.DoubleDot});
    try testTokenization(gpa, "1..2", &[_]Token.Tag{ .Int, .DoubleDot, .Int });
    try testTokenization(gpa, "...", &[_]Token.Tag{.TripleDot});
    try testTokenization(gpa, "-", &[_]Token.Tag{.OpUnaryMinus});
    try testTokenization(gpa, "-42", &[_]Token.Tag{.Int});
    try testTokenization(gpa, "1e10", &[_]Token.Tag{.Float});
    try testTokenization(gpa, "_ident", &[_]Token.Tag{.NamedUnderscore});
    try testTokenization(gpa, "1..2", &[_]Token.Tag{ .Int, .DoubleDot, .Int });
    try testTokenization(gpa, "3...4", &[_]Token.Tag{ .Int, .TripleDot, .Int });
    try testTokenization(gpa, "1. .2", &[_]Token.Tag{ .Int, .Dot, .DotInt });
    try testTokenization(gpa, "1.2.3", &[_]Token.Tag{ .Float, .NoSpaceDotInt });
    try testTokenization(gpa, "match", &[_]Token.Tag{.KwMatch});
    try testTokenization(gpa, "var", &[_]Token.Tag{.KwVar});
    try testTokenization(gpa, "{a, b}", &[_]Token.Tag{ .OpenCurly, .LowerIdent, .Comma, .LowerIdent, .CloseCurly });
    try testTokenization(gpa, "\"abc\"", &[_]Token.Tag{ .StringStart, .StringPart, .StringEnd });
    try testTokenization(gpa, "\"a${b}c\"", &[_]Token.Tag{
        .StringStart,
        .StringPart,
        .OpenStringInterpolation,
        .LowerIdent,
        .CloseStringInterpolation,
        .StringPart,
        .StringEnd,
    });
    try testTokenization(
        gpa,
        \\"""abc
    ,
        &[_]Token.Tag{ .MultilineStringStart, .StringPart },
    );
    try testTokenization(
        gpa,
        \\"""abc
        \\"""def
    ,
        &[_]Token.Tag{ .MultilineStringStart, .StringPart, .MultilineStringStart, .StringPart },
    );
    try testTokenization(
        gpa,
        \\"""abc"""
    ,
        &[_]Token.Tag{ .MultilineStringStart, .StringPart },
    );
    try testTokenization(
        gpa,
        \\"""a${b}c
        \\"""def
    ,
        &[_]Token.Tag{
            .MultilineStringStart,
            .StringPart,
            .OpenStringInterpolation,
            .LowerIdent,
            .CloseStringInterpolation,
            .StringPart,
            .MultilineStringStart,
            .StringPart,
        },
    );

    // Test new \\ multiline string syntax
    try testTokenization(
        gpa,
        "\\\\hello",
        &[_]Token.Tag{ .MultilineStringStart, .StringPart },
    );
    try testTokenization(
        gpa,
        "\\\\hello\n\\\\world",
        &[_]Token.Tag{ .MultilineStringStart, .StringPart, .MultilineStringStart, .StringPart },
    );
    try testTokenization(
        gpa,
        "\\\\a${b}c",
        &[_]Token.Tag{
            .MultilineStringStart,
            .StringPart,
            .OpenStringInterpolation,
            .LowerIdent,
            .CloseStringInterpolation,
            .StringPart,
        },
    );

    // Test dollar sign prefix for reusable identifiers
    try testTokenization(gpa, "$foo", &[_]Token.Tag{.LowerIdent});
    try testTokenization(gpa, "$Foo", &[_]Token.Tag{.UpperIdent});
    try testTokenization(gpa, "$foo123", &[_]Token.Tag{.LowerIdent});
    try testTokenization(gpa, "$", &[_]Token.Tag{.MalformedUnknownToken});
    try testTokenization(gpa, "$123", &[_]Token.Tag{ .MalformedUnknownToken, .Int });
    try testTokenization(gpa, "$ ", &[_]Token.Tag{.MalformedUnknownToken});
    try testTokenization(gpa, "$foo $bar", &[_]Token.Tag{ .LowerIdent, .LowerIdent });

    // Carriage return handling
    try testTokenization(gpa, "a\r\nb", &[_]Token.Tag{ .LowerIdent, .LowerIdent }); // Windows line ending
    try testTokenization(gpa, "a\rb", &[_]Token.Tag{ .LowerIdent, .LowerIdent }); // Standalone CR (will emit diagnostic)

    // Octal, binary, and hex numbers
    try testTokenization(gpa, "0o755", &[_]Token.Tag{.Int}); // Octal
    try testTokenization(gpa, "0O755", &[_]Token.Tag{.Int}); // Uppercase octal (will emit warning)
    try testTokenization(gpa, "0b1010", &[_]Token.Tag{.Int}); // Binary
    try testTokenization(gpa, "0B1010", &[_]Token.Tag{.Int}); // Uppercase binary (will emit warning)
    try testTokenization(gpa, "0x1A", &[_]Token.Tag{.Int}); // Hex
    try testTokenization(gpa, "0X1A", &[_]Token.Tag{.Int}); // Uppercase hex (will emit warning)
}

test "tokenizer with invalid UTF-8" {
    const gpa = std.testing.allocator;

    // Invalid UTF-8 start byte
    {
        const invalid_utf8 = [_]u8{ '"', 'H', 'e', 'l', 'l', 'o', ' ', 0xFF, ' ', 'w', 'o', 'r', 'l', 'd', '"' };
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, &invalid_utf8, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        // Should have reported InvalidUtf8InSource
        const messages = tokenizer.cursor.messages[0..tokenizer.cursor.message_count];
        try std.testing.expect(messages.len > 0);
        try std.testing.expectEqual(Diagnostic.Tag.InvalidUtf8InSource, messages[0].tag);
    }

    // Incomplete UTF-8 sequence at end
    {
        const incomplete_utf8 = [_]u8{ '"', 'H', 'e', 'l', 'l', 'o', ' ', 0xC3, '"' }; // 0xC3 expects another byte
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, &incomplete_utf8, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        // Should have reported InvalidUtf8InSource
        const messages = tokenizer.cursor.messages[0..tokenizer.cursor.message_count];
        try std.testing.expect(messages.len > 0);
        try std.testing.expectEqual(Diagnostic.Tag.InvalidUtf8InSource, messages[0].tag);
    }
}

test "non-printable characters in string literal" {
    const gpa = std.testing.allocator;

    // Non-printable ASCII
    {
        const non_printable = [_]u8{ '"', 'H', 'e', 'l', 'l', 'o', '\x01', 'w', 'o', 'r', 'l', 'd', '"' }; // 0x01 is SOH (non-printable)
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, &non_printable, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        // Should have reported NonPrintableUnicodeInStrLiteral
        const messages = tokenizer.cursor.messages[0..tokenizer.cursor.message_count];
        try std.testing.expect(messages.len > 0);
        try std.testing.expectEqual(Diagnostic.Tag.NonPrintableUnicodeInStrLiteral, messages[0].tag);
    }

    // Non-printable (but valid) non-ASCII Unicode characters
    {
        const control_char = [_]u8{ '"', 'H', 'e', 'l', 'l', 'o', ' ', 0xC2, 0x80, ' ', 'w', 'o', 'r', 'l', 'd', '"' }; // U+0080 (C1 control)
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, &control_char, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        const messages = tokenizer.cursor.messages[0..tokenizer.cursor.message_count];
        try std.testing.expect(messages.len > 0);
        try std.testing.expectEqual(Diagnostic.Tag.NonPrintableUnicodeInStrLiteral, messages[0].tag);
    }

    // No errors should be reported for these
    {
        const valid_chars = [_]u8{ '"', 'H', 'e', 'l', 'l', 'o', '\t', ' ', 'w', 'o', 'r', 'l', 'd', '"' };
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, &valid_chars, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        const messages = tokenizer.cursor.messages[0..tokenizer.cursor.message_count];
        try std.testing.expect(messages.len == 0);
    }
}

test "dollar sign in middle of identifier" {
    const gpa = std.testing.allocator;

    // Dollar sign in the middle of an identifier - foo$bar
    {
        const source = "foo$bar";
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, source, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        // Should have reported DollarInMiddleOfIdentifier
        const messages = tokenizer.cursor.messages[0..tokenizer.cursor.message_count];
        try std.testing.expect(messages.len > 0);
        try std.testing.expectEqual(Diagnostic.Tag.DollarInMiddleOfIdentifier, messages[0].tag);

        // Should tokenize as a valid LowerIdent (but with a warning)
        const token_tags = tokenizer.output.tokens.items(.tag);
        try std.testing.expect(token_tags.len >= 2); // At least the identifier and EOF
        try std.testing.expectEqual(Token.Tag.LowerIdent, token_tags[0]);
    }

    // Dollar sign at the end of an identifier - foo$
    {
        const source = "foo$";
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, source, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        // Should have reported DollarInMiddleOfIdentifier
        const messages = tokenizer.cursor.messages[0..tokenizer.cursor.message_count];
        try std.testing.expect(messages.len > 0);
        try std.testing.expectEqual(Diagnostic.Tag.DollarInMiddleOfIdentifier, messages[0].tag);
    }

    // Multiple dollar signs in identifier - foo$bar$baz
    {
        const source = "foo$bar$baz";
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, source, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        // Should have reported multiple DollarInMiddleOfIdentifier warnings
        const messages = tokenizer.cursor.messages[0..tokenizer.cursor.message_count];
        try std.testing.expect(messages.len >= 2);
        try std.testing.expectEqual(Diagnostic.Tag.DollarInMiddleOfIdentifier, messages[0].tag);
        try std.testing.expectEqual(Diagnostic.Tag.DollarInMiddleOfIdentifier, messages[1].tag);
    }

    // Dollar at the start is OK - no warning
    {
        const source = "$foo";
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, source, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        // Should NOT have any warnings
        const messages = tokenizer.cursor.messages[0..tokenizer.cursor.message_count];
        try std.testing.expect(messages.len == 0);

        // Should tokenize as a valid LowerIdent
        const token_tags = tokenizer.output.tokens.items(.tag);
        try std.testing.expect(token_tags.len >= 2);
        try std.testing.expectEqual(Token.Tag.LowerIdent, token_tags[0]);
    }
}

test "tokenizer invariants with various token types" {
    const gpa = std.testing.allocator;

    // Test various token types that should pass invariant checks
    // These are carefully chosen to NOT produce any diagnostic messages

    // Float tokens
    try checkTokenizerInvariants(gpa, "3.14", false);
    try checkTokenizerInvariants(gpa, "-3.14", false);

    // Single quote tokens (various lengths to cover different UTF-8 sizes)
    try checkTokenizerInvariants(gpa, "'A'", false); // length 3: ASCII
    try checkTokenizerInvariants(gpa, "'\\\\'", false); // length 4: escaped backslash
    try checkTokenizerInvariants(gpa, "'Å'", false); // length 4: 2-byte UTF-8 (Latin Extended)
    try checkTokenizerInvariants(gpa, "'Ἀ'", false); // length 5: 3-byte UTF-8 (Greek Extended)
    try checkTokenizerInvariants(gpa, "'𐙝'", false); // length 6: 4-byte UTF-8 (Emoji-like)
    try checkTokenizerInvariants(gpa, "'\\u(0041)'", false); // length >6: Unicode escape

    // String tokens
    try checkTokenizerInvariants(gpa, "\"hello\"", false);
    try checkTokenizerInvariants(gpa, "\"a${b}c\"", false);

    // Multiline strings
    try checkTokenizerInvariants(gpa, "\"\"\"hello\"\"\"", false);
    try checkTokenizerInvariants(gpa, "\\\\hello", false);

    // Identifiers
    try checkTokenizerInvariants(gpa, "$foo", false);
    try checkTokenizerInvariants(gpa, "$Foo", false);
    try checkTokenizerInvariants(gpa, "foo", false);
    try checkTokenizerInvariants(gpa, "Foo", false);

    // Keywords
    try checkTokenizerInvariants(gpa, "if", false);
    try checkTokenizerInvariants(gpa, "else", false);
    try checkTokenizerInvariants(gpa, "match", false);

    // Various operators
    try checkTokenizerInvariants(gpa, "+", false);
    try checkTokenizerInvariants(gpa, "-", false);
    try checkTokenizerInvariants(gpa, "*", false);
    try checkTokenizerInvariants(gpa, "=", false);
    try checkTokenizerInvariants(gpa, "==", false);

    // Underscore and named underscore
    try checkTokenizerInvariants(gpa, "_", false);
    try checkTokenizerInvariants(gpa, "_foo", false);

    // Dot identifiers and dot int
    try checkTokenizerInvariants(gpa, ".foo", false);
    try checkTokenizerInvariants(gpa, ".Foo", false);
    try checkTokenizerInvariants(gpa, ".1", false);

    // No space dot tokens (after identifier)
    try checkTokenizerInvariants(gpa, "x.foo", false);
    try checkTokenizerInvariants(gpa, "x.Foo", false);
    try checkTokenizerInvariants(gpa, "x.1", false);

    // Opaque names
    try checkTokenizerInvariants(gpa, "@Foo", false);

    // Negative hex int
    try checkTokenizerInvariants(gpa, "-0x1", false);
    try checkTokenizerInvariants(gpa, "-0xABC", false);

    // Positive hex int (various lengths)
    try checkTokenizerInvariants(gpa, "0x1", false);
    try checkTokenizerInvariants(gpa, "0xABCD", false);

    // Exclamation mark and not equals
    try checkTokenizerInvariants(gpa, "!", false);
    try checkTokenizerInvariants(gpa, "!=", false);

    // Binary minus vs unary minus context
    try checkTokenizerInvariants(gpa, "a - b", false);
    try checkTokenizerInvariants(gpa, "a-b", false);
}

test "additional operators" {
    const gpa = std.testing.allocator;

    // Pizza operator |>
    try testTokenization(gpa, "|>", &[_]Token.Tag{.OpPizza});
    try testTokenization(gpa, "a |> b", &[_]Token.Tag{ .LowerIdent, .OpPizza, .LowerIdent });

    // Pipe operator |
    try testTokenization(gpa, "|", &[_]Token.Tag{.OpBar});

    // Percent operator %
    try testTokenization(gpa, "%", &[_]Token.Tag{.OpPercent});
    try testTokenization(gpa, "a % b", &[_]Token.Tag{ .LowerIdent, .OpPercent, .LowerIdent });

    // Caret operator ^
    try testTokenization(gpa, "^", &[_]Token.Tag{.OpCaret});
    try testTokenization(gpa, "a ^ b", &[_]Token.Tag{ .LowerIdent, .OpCaret, .LowerIdent });

    // Single backslash operator
    try testTokenization(gpa, "\\", &[_]Token.Tag{.OpBackslash});

    // Double slash //
    try testTokenization(gpa, "//", &[_]Token.Tag{.OpDoubleSlash});
    try testTokenization(gpa, "a // b", &[_]Token.Tag{ .LowerIdent, .OpDoubleSlash, .LowerIdent });

    // Slash /
    try testTokenization(gpa, "/", &[_]Token.Tag{.OpSlash});
    try testTokenization(gpa, "a / b", &[_]Token.Tag{ .LowerIdent, .OpSlash, .LowerIdent });
}

test "malformed unicode identifiers" {
    const gpa = std.testing.allocator;

    // Dot followed by unicode character - malformed dot unicode ident
    // Japanese character "日" = 0xE6 0x97 0xA5 in UTF-8
    {
        const source = ".日";
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, source, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        const token_tags = tokenizer.output.tokens.items(.tag);
        try std.testing.expect(token_tags.len >= 2);
        try std.testing.expectEqual(Token.Tag.MalformedDotUnicodeIdent, token_tags[0]);
    }

    // No-space dot followed by unicode character
    {
        const source = "foo.日本";
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, source, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        const token_tags = tokenizer.output.tokens.items(.tag);
        // Should be LowerIdent followed by MalformedNoSpaceDotUnicodeIdent or similar
        try std.testing.expect(token_tags.len >= 2);
        try std.testing.expectEqual(Token.Tag.LowerIdent, token_tags[0]);
    }

    // Unicode character in lower identifier that starts with ASCII
    {
        const source = "foo日bar";
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, source, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        const token_tags = tokenizer.output.tokens.items(.tag);
        try std.testing.expect(token_tags.len >= 1);
        // Should tokenize as MalformedUnicodeIdent
        try std.testing.expectEqual(Token.Tag.MalformedUnicodeIdent, token_tags[0]);
    }
}

test "incomplete UTF-8 at end of input" {
    const gpa = std.testing.allocator;

    // Incomplete UTF-8 sequence at end of string literal
    // 0xC3 is a 2-byte UTF-8 start byte, but we only have 1 byte
    // The consumeNextCharToEof function will be called during string processing
    {
        const source = [_]u8{ '"', 'a', 0xC3 };
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, &source, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        // Should have InvalidUtf8InSource diagnostic
        const messages = tokenizer.cursor.messages[0..tokenizer.cursor.message_count];
        try std.testing.expect(messages.len > 0);
        try std.testing.expectEqual(Diagnostic.Tag.InvalidUtf8InSource, messages[0].tag);
    }
}

test "invalid unicode escape sequence in string" {
    const gpa = std.testing.allocator;

    // Invalid unicode escape - non-hex characters in \u(...)
    {
        const source = "\"\\u(GHIJ)\"";
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, source, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        // Should have InvalidUnicodeEscapeSequence diagnostic
        const messages = tokenizer.cursor.messages[0..tokenizer.cursor.message_count];
        try std.testing.expect(messages.len > 0);
        try std.testing.expectEqual(Diagnostic.Tag.InvalidUnicodeEscapeSequence, messages[0].tag);
    }

    // Incomplete unicode escape - no closing paren
    {
        const source = "\"\\u(1234\"";
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, source, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        // Should have InvalidUnicodeEscapeSequence diagnostic
        const messages = tokenizer.cursor.messages[0..tokenizer.cursor.message_count];
        try std.testing.expect(messages.len > 0);
        try std.testing.expectEqual(Diagnostic.Tag.InvalidUnicodeEscapeSequence, messages[0].tag);
    }

    // Empty unicode escape \u()
    {
        const source = "\"\\u()\"";
        var diagnostics: [10]Diagnostic = undefined;

        var env = try CommonEnv.init(gpa, try gpa.dupe(u8, ""));
        defer env.deinit(gpa);

        var tokenizer = try Tokenizer.init(&env, gpa, source, &diagnostics);
        defer tokenizer.deinit(gpa);
        try tokenizer.tokenize(gpa);

        // Should have InvalidUnicodeEscapeSequence diagnostic
        const messages = tokenizer.cursor.messages[0..tokenizer.cursor.message_count];
        try std.testing.expect(messages.len > 0);
        try std.testing.expectEqual(Diagnostic.Tag.InvalidUnicodeEscapeSequence, messages[0].tag);
    }
}