Implement the memory64 proposal in Wasmtime

build.rs

@@ -34,6 +34,7 @@ fn main() -> anyhow::Result<()> {
             test_directory_module(out, "tests/misc_testsuite/module-linking", strategy)?;
             test_directory_module(out, "tests/misc_testsuite/simd", strategy)?;
             test_directory_module(out, "tests/misc_testsuite/threads", strategy)?;
+            test_directory_module(out, "tests/misc_testsuite/memory64", strategy)?;
             Ok(())
         })?;
 
@@ -53,6 +54,7 @@ fn main() -> anyhow::Result<()> {
                     "tests/spec_testsuite/proposals/bulk-memory-operations",
                     strategy,
                 )?;
+                test_directory_module(out, "tests/spec_testsuite/proposals/memory64", strategy)?;
             } else {
                 println!(
                     "cargo:warning=The spec testsuite is disabled. To enable, run `git submodule \
@@ -157,7 +159,7 @@ fn write_testsuite_tests(
 
     writeln!(out, "#[test]")?;
     // Ignore when using QEMU for running tests (limited memory).
-    if ignore(testsuite, &testname, strategy) || (pooling && platform_is_emulated()) {
+    if ignore(testsuite, &testname, strategy) {
         writeln!(out, "#[ignore]")?;
     }
 
@@ -213,7 +215,3 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool {
 fn platform_is_s390x() -> bool {
     env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "s390x"
 }
-
-fn platform_is_emulated() -> bool {
-    env::var("WASMTIME_TEST_NO_HOG_MEMORY").unwrap_or_default() == "1"
-}

cranelift/codegen/src/ir/immediates.rs

@@ -291,6 +291,12 @@ impl From<Uimm32> for u32 {
     }
 }
 
+impl From<Uimm32> for u64 {
+    fn from(val: Uimm32) -> u64 {
+        val.0.into()
+    }
+}
+
 impl From<Uimm32> for i64 {
     fn from(val: Uimm32) -> i64 {
         i64::from(val.0)

cranelift/codegen/src/legalizer/heap.rs

@@ -25,7 +25,7 @@ pub fn expand_heap_addr(
             imm,
         } => {
             debug_assert_eq!(opcode, ir::Opcode::HeapAddr);
-            (heap, arg, imm.into())
+            (heap, arg, u64::from(imm))
         }
         _ => panic!("Wanted heap_addr: {}", func.dfg.display_inst(inst, None)),
     };
@@ -53,11 +53,10 @@ fn dynamic_addr(
     inst: ir::Inst,
     heap: ir::Heap,
     offset: ir::Value,
-    access_size: u32,
+    access_size: u64,
     bound_gv: ir::GlobalValue,
     func: &mut ir::Function,
 ) {
-    let access_size = u64::from(access_size);
     let offset_ty = func.dfg.value_type(offset);
     let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
     let min_size = func.heaps[heap].min_size.into();
@@ -113,12 +112,11 @@ fn static_addr(
     inst: ir::Inst,
     heap: ir::Heap,
     mut offset: ir::Value,
-    access_size: u32,
+    access_size: u64,
     bound: u64,
     func: &mut ir::Function,
     cfg: &mut ControlFlowGraph,
 ) {
-    let access_size = u64::from(access_size);
     let offset_ty = func.dfg.value_type(offset);
     let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
     let mut pos = FuncCursor::new(func).at_inst(inst);

cranelift/wasm/src/code_translator.rs

@@ -2164,10 +2164,6 @@ fn prepare_addr<FE: FuncEnvironment + ?Sized>(
     environ: &mut FE,
 ) -> WasmResult<(MemFlags, Value, Offset32)> {
     let addr = state.pop1();
-    // This function will need updates for 64-bit memories
-    debug_assert_eq!(builder.func.dfg.value_type(addr), I32);
-    let offset = u32::try_from(memarg.offset).unwrap();
-
     let heap = state.get_heap(builder.func, memarg.memory, environ)?;
     let offset_guard_size: u64 = builder.func.heaps[heap].offset_guard_size.into();
 
@@ -2222,25 +2218,65 @@ fn prepare_addr<FE: FuncEnvironment + ?Sized>(
     // offsets we're checking here are zero. This means that we'll hit the fast
     // path and emit zero conditional traps for bounds checks
     let adjusted_offset = if offset_guard_size == 0 {
-        u64::from(offset) + u64::from(access_size)
+        memarg.offset.saturating_add(u64::from(access_size))
     } else {
         assert!(access_size < 1024);
-        cmp::max(u64::from(offset) / offset_guard_size * offset_guard_size, 1)
+        cmp::max(memarg.offset / offset_guard_size * offset_guard_size, 1)
     };
     debug_assert!(adjusted_offset > 0); // want to bounds check at least 1 byte
-    let check_size = u32::try_from(adjusted_offset).unwrap_or(u32::MAX);
-    let base = builder
-        .ins()
-        .heap_addr(environ.pointer_type(), heap, addr, check_size);
-
-    // Native load/store instructions take a signed `Offset32` immediate, so adjust the base
-    // pointer if necessary.
-    let (addr, offset) = if offset > i32::MAX as u32 {
-        // Offset doesn't fit in the load/store instruction.
-        let adj = builder.ins().iadd_imm(base, i64::from(i32::MAX) + 1);
-        (adj, (offset - (i32::MAX as u32 + 1)) as i32)
-    } else {
-        (base, offset as i32)
+    let (addr, offset) = match u32::try_from(adjusted_offset) {
+        // If our adjusted offset fits within a u32, then we can place the
+        // entire offset into the offset of the `heap_addr` instruction. After
+        // the `heap_addr` instruction, though, we need to factor the the offset
+        // into the returned address. This is either an immediate if the offset
+        // further fits within `i32`, or a manual add instruction otherwise.
+        //
+        // Note that native instructions take a signed offset hence the switch
+        // to i32. Note also the lack of overflow checking in the offset
+        // addition, which should be ok since if `heap_addr` passed we're
+        // guaranteed that this won't overflow.
+        Ok(adjusted_offset) => {
+            let base = builder
+                .ins()
+                .heap_addr(environ.pointer_type(), heap, addr, adjusted_offset);
+            match i32::try_from(memarg.offset) {
+                Ok(val) => (base, val),
+                Err(_) => {
+                    let adj = builder.ins().iadd_imm(base, memarg.offset as i64);
+                    (adj, 0)
+                }
+            }
+        }
+
+        // If the adjusted offset doesn't fit within a u32, then this gets
+        // pessimized a fair amount. We can't pass the fixed-sized offset to
+        // the `heap_addr` instruction, so we need to fold the offset into the
+        // address itself. In doing so though we need to check for overflow
+        // because that would mean the address is out-of-bounds.
+        //
+        // Once we have the effective address, offset already folded in, then
+        // `heap_addr` is used to verify that the address is indeed in-bounds.
+        // The access size of the `heap_addr` is what we were passed in from
+        // above.
+        //
+        // Note that this is generating what's likely to be at least two
+        // branches, one for the overflow and one for the bounds check itself.
+        // For now though that should hopefully be ok since 4gb+ offsets are
+        // relatively odd/rare.
+        Err(_) => {
+            let index_type = builder.func.heaps[heap].index_type;
+            let offset = builder.ins().iconst(index_type, memarg.offset as i64);
+            let (addr, overflow) = builder.ins().iadd_ifcout(addr, offset);
+            builder.ins().trapif(
+                environ.unsigned_add_overflow_condition(),
+                overflow,
+                ir::TrapCode::HeapOutOfBounds,
+            );
+            let base = builder
+                .ins()
+                .heap_addr(environ.pointer_type(), heap, addr, access_size);
+            (base, 0)
+        }
     };
 
     // Note that we don't set `is_aligned` here, even if the load instruction's

cranelift/wasm/src/environ/dummy.rs

@@ -652,6 +652,10 @@ impl<'dummy_environment> FuncEnvironment for DummyFuncEnvironment<'dummy_environ
     ) -> WasmResult<ir::Value> {
         Ok(pos.ins().iconst(I32, 0))
     }
+
+    fn unsigned_add_overflow_condition(&self) -> ir::condcodes::IntCC {
+        unimplemented!()
+    }
 }
 
 impl TargetEnvironment for DummyEnvironment {
@@ -792,7 +796,7 @@ impl<'data> ModuleEnvironment<'data> for DummyEnvironment {
         &mut self,
         _memory_index: MemoryIndex,
         _base: Option<GlobalIndex>,
-        _offset: u32,
+        _offset: u64,
         _data: &'data [u8],
     ) -> WasmResult<()> {
         // We do nothing

cranelift/wasm/src/environ/spec.rs

@@ -697,6 +697,10 @@ pub trait FuncEnvironment: TargetEnvironment {
     ) -> WasmResult<()> {
         Ok(())
     }
+
+    /// Returns the target ISA's condition to check for unsigned addition
+    /// overflowing.
+    fn unsigned_add_overflow_condition(&self) -> ir::condcodes::IntCC;
 }
 
 /// An object satisfying the `ModuleEnvironment` trait can be passed as argument to the
@@ -995,7 +999,7 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
         &mut self,
         memory_index: MemoryIndex,
         base: Option<GlobalIndex>,
-        offset: u32,
+        offset: u64,
         data: &'data [u8],
     ) -> WasmResult<()>;
 

cranelift/wasm/src/sections_translator.rs

@@ -54,11 +54,11 @@ fn entity_type(
 }
 
 fn memory(ty: MemoryType) -> Memory {
-    assert!(!ty.memory64);
     Memory {
-        minimum: ty.initial.try_into().unwrap(),
-        maximum: ty.maximum.map(|i| i.try_into().unwrap()),
+        minimum: ty.initial,
+        maximum: ty.maximum,
         shared: ty.shared,
+        memory64: ty.memory64,
     }
 }
 
@@ -420,7 +420,8 @@ pub fn parse_data_section<'data>(
             } => {
                 let mut init_expr_reader = init_expr.get_binary_reader();
                 let (base, offset) = match init_expr_reader.read_operator()? {
-                    Operator::I32Const { value } => (None, value as u32),
+                    Operator::I32Const { value } => (None, value as u64),
+                    Operator::I64Const { value } => (None, value as u64),
                     Operator::GlobalGet { global_index } => {
                         (Some(GlobalIndex::from_u32(global_index)), 0)
                     }

cranelift/wasm/src/translation_utils.rs

@@ -226,11 +226,13 @@ pub enum TableElementType {
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
 pub struct Memory {
     /// The minimum number of pages in the memory.
-    pub minimum: u32,
+    pub minimum: u64,
     /// The maximum number of pages in the memory.
-    pub maximum: Option<u32>,
+    pub maximum: Option<u64>,
     /// Whether the memory may be shared between multiple threads.
     pub shared: bool,
+    /// Whether or not this is a 64-bit memory
+    pub memory64: bool,
 }
 
 /// WebAssembly event.

crates/c-api/include/doc-wasm.h

@@ -594,11 +594,17 @@
  *
  * The caller is responsible for deallocating the returned type.
  *
+ * For compatibility with memory64 it's recommended to use
+ * #wasmtime_memorytype_new instead.
+ *
  * \fn const wasm_limits_t* wasm_memorytype_limits(const wasm_memorytype_t *);
  * \brief Returns the limits of this memory.
  *
  * The returned #wasm_limits_t is owned by the #wasm_memorytype_t parameter, the
  * caller should not deallocate it.
+ *
+ * For compatibility with memory64 it's recommended to use
+ * #wasmtime_memorytype_maximum or #wasmtime_memorytype_minimum instead.
  */
 
 /**

crates/c-api/include/wasmtime/memory.h

@@ -16,6 +16,39 @@
 extern "C" {
 #endif
 
+/**
+ * \brief Creates a new memory type from the specified parameters.
+ *
+ * Note that this function is preferred over #wasm_memorytype_new for
+ * compatibility with the memory64 proposal.
+ */
+WASM_API_EXTERN wasm_memorytype_t *wasmtime_memorytype_new(uint64_t min, bool max_present, uint64_t max, bool is_64);
+
+/**
+ * \brief Returns the minimum size, in pages, of the specified memory type.
+ *
+ * Note that this function is preferred over #wasm_memorytype_limits for
+ * compatibility with the memory64 proposal.
+ */
+WASM_API_EXTERN uint64_t wasmtime_memorytype_minimum(const wasm_memorytype_t *ty);
+
+/**
+ * \brief Returns the maximum size, in pages, of the specified memory type.
+ *
+ * If this memory type doesn't have a maximum size listed then `false` is
+ * returned. Otherwise `true` is returned and the `max` pointer is filled in
+ * with the specified maximum size, in pages.
+ *
+ * Note that this function is preferred over #wasm_memorytype_limits for
+ * compatibility with the memory64 proposal.
+ */
+WASM_API_EXTERN bool wasmtime_memorytype_maximum(const wasm_memorytype_t *ty, uint64_t *max);
+
+/**
+ * \brief Returns whether this type of memory represents a 64-bit memory.
+ */
+WASM_API_EXTERN bool wasmtime_memorytype_is64(const wasm_memorytype_t *ty);
+
 /**
  * \brief Creates a new WebAssembly linear memory
  *
@@ -59,7 +92,7 @@ WASM_API_EXTERN size_t wasmtime_memory_data_size(
 /**
  * \brief Returns the length, in WebAssembly pages, of this linear memory
  */
-WASM_API_EXTERN uint32_t wasmtime_memory_size(
+WASM_API_EXTERN uint64_t wasmtime_memory_size(
     const wasmtime_context_t *store,
     const wasmtime_memory_t *memory
 );
@@ -79,8 +112,8 @@ WASM_API_EXTERN uint32_t wasmtime_memory_size(
 WASM_API_EXTERN wasmtime_error_t *wasmtime_memory_grow(
     wasmtime_context_t *store,
     const wasmtime_memory_t *memory,
-    uint32_t delta,
-    uint32_t *prev_size
+    uint64_t delta,
+    uint64_t *prev_size
 );
 
 #ifdef __cplusplus

crates/c-api/src/memory.rs

@@ -2,6 +2,7 @@ use crate::{
     handle_result, wasm_extern_t, wasm_memorytype_t, wasm_store_t, wasmtime_error_t, CStoreContext,
     CStoreContextMut,
 };
+use std::convert::TryFrom;
 use wasmtime::{Extern, Memory};
 
 #[derive(Clone)]
@@ -72,7 +73,7 @@ pub unsafe extern "C" fn wasm_memory_data_size(m: &wasm_memory_t) -> usize {
 
 #[no_mangle]
 pub unsafe extern "C" fn wasm_memory_size(m: &wasm_memory_t) -> wasm_memory_pages_t {
-    m.memory().size(m.ext.store.context())
+    u32::try_from(m.memory().size(m.ext.store.context())).unwrap()
 }
 
 #[no_mangle]
@@ -82,7 +83,7 @@ pub unsafe extern "C" fn wasm_memory_grow(
 ) -> bool {
     let memory = m.memory();
     let mut store = m.ext.store.context_mut();
-    memory.grow(&mut store, delta).is_ok()
+    memory.grow(&mut store, u64::from(delta)).is_ok()
 }
 
 #[no_mangle]
@@ -113,16 +114,16 @@ pub extern "C" fn wasmtime_memory_data_size(store: CStoreContext<'_>, mem: &Memo
 }
 
 #[no_mangle]
-pub extern "C" fn wasmtime_memory_size(store: CStoreContext<'_>, mem: &Memory) -> u32 {
+pub extern "C" fn wasmtime_memory_size(store: CStoreContext<'_>, mem: &Memory) -> u64 {
     mem.size(store)
 }
 
 #[no_mangle]
 pub extern "C" fn wasmtime_memory_grow(
     store: CStoreContextMut<'_>,
     mem: &Memory,
-    delta: u32,
-    prev_size: &mut u32,
+    delta: u64,
+    prev_size: &mut u64,
 ) -> Option<Box<wasmtime_error_t>> {
     handle_result(mem.grow(store, delta), |prev| *prev_size = prev)
 }