feat(compiler)!: Change Char internal representation (#1622)

Author: alex-snezhko

compiler/src/codegen/compcore.re

@@ -1728,7 +1728,7 @@ let allocate_bytes_uninitialized = (wasm_mod, env, size) => {
 let create_char = (wasm_mod, env, char) => {
   let uchar = List.hd @@ Utf8.decodeUtf8String(char);
   let uchar_int: int = Utf8__Uchar.toInt(uchar);
-  let grain_char = uchar_int lsl 3 lor 0b010;
+  let grain_char = uchar_int lsl 8 lor 0b010;
   Expression.Const.make(wasm_mod, const_int32(grain_char));
 };
 
@@ -2342,7 +2342,7 @@ let compile_prim1 = (wasm_mod, env, p1, arg, loc): Expression.t => {
         wasm_mod,
         Op.shl_int32,
         compiled_arg,
-        Expression.Const.make(wasm_mod, const_int32(0x3)),
+        Expression.Const.make(wasm_mod, const_int32(0x8)),
       ),
       Expression.Const.make(wasm_mod, const_int32(0b10)),
     )
@@ -2351,7 +2351,7 @@ let compile_prim1 = (wasm_mod, env, p1, arg, loc): Expression.t => {
       wasm_mod,
       Op.shr_s_int32,
       compiled_arg,
-      Expression.Const.make(wasm_mod, const_int32(0x3)),
+      Expression.Const.make(wasm_mod, const_int32(0x8)),
     )
   | Not =>
     /* Flip the first bit */

compiler/test/__snapshots__/chars.200d9e1a.0.snapshot

@@ -33,7 +33,7 @@ chars › char4
    (block $cleanup_locals.2 (result i32)
     (local.set $0
      (block $compile_block.1 (result i32)
-      (i32.const 522)
+      (i32.const 16642)
      )
     )
     (local.get $0)

compiler/test/__snapshots__/chars.259e330c.0.snapshot

@@ -33,7 +33,7 @@ chars › char2
    (block $cleanup_locals.2 (result i32)
     (local.set $0
      (block $compile_block.1 (result i32)
-      (i32.const 522)
+      (i32.const 16642)
      )
     )
     (local.get $0)

compiler/test/__snapshots__/chars.27fb7f30.0.snapshot

@@ -33,7 +33,7 @@ chars › char8
    (block $cleanup_locals.2 (result i32)
     (local.set $0
      (block $compile_block.1 (result i32)
-      (i32.const 80194)
+      (i32.const 2566146)
      )
     )
     (local.get $0)

compiler/test/__snapshots__/chars.51010573.0.snapshot

@@ -33,7 +33,7 @@ chars › char7
    (block $cleanup_locals.2 (result i32)
     (local.set $0
      (block $compile_block.1 (result i32)
-      (i32.const 1022450)
+      (i32.const 32718338)
      )
     )
     (local.get $0)

compiler/test/__snapshots__/chars.7e0f68db.0.snapshot

@@ -33,7 +33,7 @@ chars › char6
    (block $cleanup_locals.2 (result i32)
     (local.set $0
      (block $compile_block.1 (result i32)
-      (i32.const 1025402)
+      (i32.const 32812802)
      )
     )
     (local.get $0)

compiler/test/__snapshots__/chars.af4b3613.0.snapshot

@@ -33,7 +33,7 @@ chars › char5
    (block $cleanup_locals.2 (result i32)
     (local.set $0
      (block $compile_block.1 (result i32)
-      (i32.const 522)
+      (i32.const 16642)
      )
     )
     (local.get $0)

compiler/test/__snapshots__/chars.e1cac8cd.0.snapshot

@@ -33,7 +33,7 @@ chars › char3
    (block $cleanup_locals.2 (result i32)
     (local.set $0
      (block $compile_block.1 (result i32)
-      (i32.const 522)
+      (i32.const 16642)
      )
     )
     (local.get $0)

docs/contributor/data_representations.md

@@ -36,7 +36,7 @@ To avoid overwriting data, we shift simple numbers to the left by 1 bit, then se
 
 ### Chars
 
-The Grain `Char` type is represented as a tagged Unicode scalar value. They're similar to simple numbers, though they use a full 3-bit tag, `0b010`. Unicode scalar values exist in the range 0x0-10FFFF, which means it only takes 21 bits to store a USV—that leaves plenty of room for our 3-bit tag, and we cover the full spectrum of USVs. To tag a USV, we shift the value left by 3 bits and set the last 3 bits to our tag value, `0b010`. This means that Grain chars are stored as `8n + 2`, where `n` is the USV. Just like numbers, there are some tricks we can do to avoid untagging and retagging when manipulating chars. For example, the successor of a char can be found by just adding 8: `8(n + 1) + 2` is `(8n + 2) + 8`.
+The Grain `Char` type is represented as a tagged Unicode scalar value. They're similar to simple numbers, though they use a full 3-bit tag, `0b010`. Unicode scalar values exist in the range 0x0-10FFFF, which means it only takes 21 bits to store a USV—that leaves plenty of room for our 3-bit tag, and we cover the full spectrum of USVs. To tag a USV, we shift the value left by 8 bits and set the last 3 bits to our tag value, `0b010`. This means that Grain chars are stored as `(2^8)n + 2`, where `n` is the USV. Just like numbers, there are some tricks we can do to avoid untagging and retagging when manipulating chars.
 
 ## Structure of Heap-Allocated Data
 

stdlib/char.gr

@@ -10,7 +10,6 @@
 
 import WasmI32 from "runtime/unsafe/wasmi32"
 import Errors from "runtime/unsafe/errors"
-import Tags from "runtime/unsafe/tags"
 import {
   tagSimpleNumber,
   tagChar,
@@ -92,10 +91,10 @@ export let fromCode = (usv: Number) => {
 
   // Here we use a math trick to avoid fully untagging and retagging.
   // Simple numbers are represented as 2n + 1 and chars are represented as
-  // 8n + 2. Quick reminder that shifting left is the equivalent of multiplying
-  // by 2, and that _GRAIN_CHAR_TAG_TYPE is equal to 2:
-  // 4(2n + 1) - 2 = 8n + 2
-  let char = (usv << 2n) - Tags._GRAIN_CHAR_TAG_TYPE
+  // (2^8)n + 2. Quick reminder that shifting left is the equivalent of multiplying
+  // by 2
+  // 2^7(2n + 1) - (2^7 - 2) = (2^8)n + 2
+  let char = (usv << 7n) - 126n
 
   WasmI32.toGrain(char): Char
 }