chore(compiler): Refactor literals in the lexer (#2030)

Co-authored-by: Oscar Spencer <oscar@grain-lang.org>

Author: phated

compiler/src/formatting/format.re

@@ -2564,7 +2564,7 @@ and get_function_name = (expr: Parsetree.expression) => {
   switch (expr.pexp_desc) {
   | PExpConstant(x) =>
     switch (x) {
-    | PConstString(str) => str
+    | PConstString({txt: str}) => str
     | _ => ""
     }
 

compiler/src/parsing/ast_helper.re

@@ -58,6 +58,15 @@ let record_pattern_info = record_pats =>
     ([], Asttypes.Closed),
   );
 
+// This normalizes STRING items in the parsetree that aren't constants so we don't need
+// to scatter them throughout the rest of the compiler.
+let normalize_string = (~loc, item) => {
+  switch (Grain_utils.Literals.conv_string(item.txt)) {
+  | Ok(i) => {loc: item.loc, txt: i}
+  | Error(msg) => raise(SyntaxError(loc, msg))
+  };
+};
+
 module Constant = {
   let bytes = b => PConstBytes(b);
   let string = s => PConstString(s);
@@ -67,25 +76,18 @@ module Constant = {
   let int16 = i => PConstInt16(i);
   let int32 = i => PConstInt32(i);
   let int64 = i => PConstInt64(i);
-  let uint8 = (is_neg, i) => PConstUint8(is_neg, i);
-  let uint16 = (is_neg, i) => PConstUint16(is_neg, i);
-  let uint32 = (is_neg, i) => PConstUint32(is_neg, i);
-  let uint64 = (is_neg, i) => PConstUint64(is_neg, i);
+  let uint8 = i => PConstUint8(i);
+  let uint16 = i => PConstUint16(i);
+  let uint32 = i => PConstUint32(i);
+  let uint64 = i => PConstUint64(i);
   let float32 = f => PConstFloat32(f);
   let float64 = f => PConstFloat64(f);
   let wasmi32 = i => PConstWasmI32(i);
   let wasmi64 = i => PConstWasmI64(i);
   let wasmf32 = f => PConstWasmF32(f);
   let wasmf64 = f => PConstWasmF64(f);
   let bigint = i => PConstBigInt(i);
-  let rational = r => {
-    let (n, d) =
-      switch (String.split_on_char('/', r)) {
-      | [n, d] => (n, d)
-      | _ => failwith("Impossible: rational literal without forward slash")
-      };
-    PConstRational(n, d);
-  };
+  let rational = r => PConstRational(r);
   let bool = b => PConstBool(b);
   let void = PConstVoid;
 };
@@ -467,14 +469,18 @@ module Toplevel = {
 
 module PrimitiveDescription = {
   let mk = (~loc, ~ident, ~name, ()) => {
-    {pprim_ident: ident, pprim_name: name, pprim_loc: loc};
+    {
+      pprim_ident: ident,
+      pprim_name: normalize_string(~loc, name),
+      pprim_loc: loc,
+    };
   };
 };
 
 module ValueDescription = {
   let mk = (~loc, ~mod_, ~name, ~alias, ~typ, ()) => {
     {
-      pval_mod: mod_,
+      pval_mod: normalize_string(~loc, mod_),
       pval_name: name,
       pval_name_alias: alias,
       pval_type: typ,
@@ -497,7 +503,11 @@ module MatchBranch = {
 
 module IncludeDeclaration = {
   let mk = (~loc, path, alias) => {
-    {pinc_alias: alias, pinc_path: path, pinc_loc: loc};
+    {
+      pinc_alias: alias,
+      pinc_path: normalize_string(~loc, path),
+      pinc_loc: loc,
+    };
   };
 };
 
@@ -545,3 +555,9 @@ module ModuleDeclaration = {
     {pmod_name: name, pmod_stmts: stmts, pmod_loc: loc};
   };
 };
+
+module Attribute = {
+  let mk = (~loc, name, args) => {
+    (name, List.map(normalize_string(~loc), args));
+  };
+};

compiler/src/parsing/ast_helper.rei

@@ -35,26 +35,26 @@ type str = loc(string);
 type loc = Location.t;
 
 module Constant: {
-  let bytes: string => constant;
-  let string: string => constant;
-  let char: string => constant;
+  let bytes: str => constant;
+  let string: str => constant;
+  let char: str => constant;
   let number: number_type => constant;
-  let int8: string => constant;
-  let int16: string => constant;
-  let int32: string => constant;
-  let int64: string => constant;
-  let uint8: (bool, string) => constant;
-  let uint16: (bool, string) => constant;
-  let uint32: (bool, string) => constant;
-  let uint64: (bool, string) => constant;
-  let float32: string => constant;
-  let float64: string => constant;
-  let wasmi32: string => constant;
-  let wasmi64: string => constant;
-  let wasmf32: string => constant;
-  let wasmf64: string => constant;
-  let bigint: string => constant;
-  let rational: string => constant;
+  let int8: str => constant;
+  let int16: str => constant;
+  let int32: str => constant;
+  let int64: str => constant;
+  let uint8: str => constant;
+  let uint16: str => constant;
+  let uint32: str => constant;
+  let uint64: str => constant;
+  let float32: str => constant;
+  let float64: str => constant;
+  let wasmi32: str => constant;
+  let wasmi64: str => constant;
+  let wasmf32: str => constant;
+  let wasmf64: str => constant;
+  let bigint: str => constant;
+  let rational: str => constant;
   let bool: bool => constant;
   let void: constant;
 };
@@ -380,3 +380,7 @@ module LambdaArgument: {
 module ModuleDeclaration: {
   let mk: (~loc: loc, str, list(toplevel_stmt)) => module_declaration;
 };
+
+module Attribute: {
+  let mk: (~loc: loc, str, list(str)) => attribute;
+};