From 071539035c887d078b1c63fd73b76294d108d9fa Mon Sep 17 00:00:00 2001 From: Frank Emrich Date: Wed, 12 Mar 2025 18:46:59 +0000 Subject: [PATCH 1/6] [pr1] base --- Cargo.toml | 2 + crates/c-api/include/wasmtime/config.h | 16 + crates/c-api/include/wasmtime/linker.h | 10 + crates/c-api/src/config.rs | 4 + crates/cli-flags/Cargo.toml | 1 + crates/cli-flags/src/lib.rs | 14 + crates/cranelift/src/func_environ.rs | 17 + crates/cranelift/src/gc/enabled.rs | 22 +- crates/cranelift/src/lib.rs | 12 +- .../src/translate/code_translator.rs | 61 +- crates/environ/src/builtin.rs | 29 + crates/environ/src/gc.rs | 9 +- crates/environ/src/lib.rs | 1 + crates/environ/src/stack_switching.rs | 78 ++ crates/environ/src/trap_encoding.rs | 16 + crates/environ/src/types.rs | 24 +- crates/environ/src/vmoffsets.rs | 127 ++++ crates/fuzzing/src/generators/config.rs | 1 + crates/misc/component-test-util/src/lib.rs | 5 +- crates/wasmtime/Cargo.toml | 8 +- crates/wasmtime/src/config.rs | 67 +- crates/wasmtime/src/engine.rs | 19 +- .../wasmtime/src/runtime/externals/global.rs | 4 + .../wasmtime/src/runtime/externals/table.rs | 5 + crates/wasmtime/src/runtime/func.rs | 243 +++--- .../src/runtime/gc/enabled/arrayref.rs | 6 + .../src/runtime/gc/enabled/structref.rs | 10 +- crates/wasmtime/src/runtime/store.rs | 146 +++- crates/wasmtime/src/runtime/type_registry.rs | 2 +- crates/wasmtime/src/runtime/types.rs | 181 ++++- crates/wasmtime/src/runtime/values.rs | 5 + crates/wasmtime/src/runtime/vm.rs | 2 + .../instance/allocator/pooling/table_pool.rs | 15 +- crates/wasmtime/src/runtime/vm/libcalls.rs | 103 +++ .../src/runtime/vm/stack_switching.rs | 705 ++++++++++++++++++ .../src/runtime/vm/stack_switching/stack.rs | 119 +++ .../runtime/vm/stack_switching/stack/dummy.rs | 75 ++ .../runtime/vm/stack_switching/stack/unix.rs | 354 +++++++++ .../vm/stack_switching/stack/unix/x86_64.rs | 86 +++ crates/wasmtime/src/runtime/vm/table.rs | 204 ++++- .../wasmtime/src/runtime/vm/traphandlers.rs | 137 ++-- .../src/runtime/vm/traphandlers/backtrace.rs | 158 +++- crates/wasmtime/src/runtime/vm/vmcontext.rs | 8 +- crates/wast-util/src/lib.rs | 7 + tests/all/main.rs | 1 + 45 files changed, 2874 insertions(+), 245 deletions(-) create mode 100644 crates/environ/src/stack_switching.rs create mode 100644 crates/wasmtime/src/runtime/vm/stack_switching.rs create mode 100644 crates/wasmtime/src/runtime/vm/stack_switching/stack.rs create mode 100644 crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs create mode 100644 crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs create mode 100644 crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs diff --git a/Cargo.toml b/Cargo.toml index 100d63bfa58c..47218b3081db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -431,6 +431,7 @@ default = [ "gc", "gc-drc", "gc-null", + "stack-switching", "winch", "pulley", @@ -489,6 +490,7 @@ gc = ["wasmtime-cli-flags/gc", "wasmtime/gc"] gc-drc = ["gc", "wasmtime/gc-drc", "wasmtime-cli-flags/gc-drc"] gc-null = ["gc", "wasmtime/gc-null", "wasmtime-cli-flags/gc-null"] pulley = ["wasmtime-cli-flags/pulley"] +stack-switching = ["wasmtime/stack-switching", "wasmtime-cli-flags/stack-switching"] # CLI subcommands for the `wasmtime` executable. See `wasmtime $cmd --help` # for more information on each subcommand. diff --git a/crates/c-api/include/wasmtime/config.h b/crates/c-api/include/wasmtime/config.h index 4788ea9bb145..8a262859cf8c 100644 --- a/crates/c-api/include/wasmtime/config.h +++ b/crates/c-api/include/wasmtime/config.h @@ -250,6 +250,22 @@ WASMTIME_CONFIG_PROP(void, wasm_wide_arithmetic, bool) #ifdef WASMTIME_FEATURE_COMPILER +/** + * \brief Configures whether the WebAssembly function references + * proposal is enabled. + * + * This setting is `false` by default. + */ +WASMTIME_CONFIG_PROP(void, wasm_function_references, bool) + +/** + * \brief Configures whether the WebAssembly stack switching + * proposal is enabled. + * + * This setting is `false` by default. + */ +WASMTIME_CONFIG_PROP(void, wasm_stack_switching, bool) + /** * \brief Configures how JIT code will be compiled. * diff --git a/crates/c-api/include/wasmtime/linker.h b/crates/c-api/include/wasmtime/linker.h index 0a793d1e6fc9..c97d824d2efd 100644 --- a/crates/c-api/include/wasmtime/linker.h +++ b/crates/c-api/include/wasmtime/linker.h @@ -66,6 +66,16 @@ WASM_API_EXTERN void wasmtime_linker_delete(wasmtime_linker_t *linker); WASM_API_EXTERN void wasmtime_linker_allow_shadowing(wasmtime_linker_t *linker, bool allow_shadowing); +/** + * \brief Configures whether the given Linker will allow unknown exports from + * command modules. + * + * By default this setting is `false`. + */ +WASM_API_EXTERN void +wasmtime_linker_allow_unknown_exports(wasmtime_linker_t *linker, + bool allow_unknown_exports); + /** * \brief Defines a new item in this linker. * diff --git a/crates/c-api/src/config.rs b/crates/c-api/src/config.rs index 455802416cd5..25198d95af54 100644 --- a/crates/c-api/src/config.rs +++ b/crates/c-api/src/config.rs @@ -137,6 +137,10 @@ pub extern "C" fn wasmtime_config_wasm_memory64_set(c: &mut wasm_config_t, enabl } #[unsafe(no_mangle)] +pub extern "C" fn wasmtime_config_wasm_stack_switching_set(c: &mut wasm_config_t, enable: bool) { + c.config.wasm_stack_switching(enable); +} + #[cfg(any(feature = "cranelift", feature = "winch"))] pub extern "C" fn wasmtime_config_strategy_set( c: &mut wasm_config_t, diff --git a/crates/cli-flags/Cargo.toml b/crates/cli-flags/Cargo.toml index 23dc34036e85..1469c9535b15 100644 --- a/crates/cli-flags/Cargo.toml +++ b/crates/cli-flags/Cargo.toml @@ -39,3 +39,4 @@ gc-null = ["gc", "wasmtime/gc-null"] threads = ["wasmtime/threads"] memory-protection-keys = ["wasmtime/memory-protection-keys"] pulley = ["wasmtime/pulley"] +stack-switching = ["wasmtime/stack-switching"] diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs index 9079d7f882bb..1e86e1f2ca4a 100644 --- a/crates/cli-flags/src/lib.rs +++ b/crates/cli-flags/src/lib.rs @@ -331,6 +331,8 @@ wasmtime_option_group! { pub trap_on_grow_failure: Option, /// Maximum execution time of wasm code before timing out (1, 2s, 100ms, etc) pub timeout: Option, + /// Size of stacks created with cont.new instructions + pub stack_switching_stack_size: Option, /// Configures support for all WebAssembly proposals implemented. pub all_proposals: Option, /// Configure support for the bulk memory proposal. @@ -366,6 +368,8 @@ wasmtime_option_group! { pub component_model_async: Option, /// Configure support for the function-references proposal. pub function_references: Option, + /// Configure support for the stack-switching proposal. + pub stack_switching: Option, /// Configure support for the GC proposal. pub gc: Option, /// Configure support for the custom-page-sizes proposal. @@ -803,6 +807,12 @@ impl CommonOptions { config.native_unwind_info(enable); } + match_feature! { + ["stack-switching" : self.wasm.stack_switching_stack_size] + size => config.stack_switching_stack_size(size), + _ => err, + } + match_feature! { ["pooling-allocator" : self.opts.pooling_allocator.or(pooling_allocator_default)] enable => { @@ -964,6 +974,9 @@ impl CommonOptions { if let Some(enable) = self.wasm.memory64.or(all) { config.wasm_memory64(enable); } + if let Some(enable) = self.wasm.stack_switching { + config.wasm_stack_switching(enable); + } if let Some(enable) = self.wasm.custom_page_sizes.or(all) { config.wasm_custom_page_sizes(enable); } @@ -994,6 +1007,7 @@ impl CommonOptions { ("gc", gc, wasm_gc) ("gc", reference_types, wasm_reference_types) ("gc", function_references, wasm_function_references) + ("stack-switching", stack_switching, wasm_stack_switching) } Ok(()) } diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 3177b0af7def..b370bfcf01b4 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -3441,3 +3441,20 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type { IndexType::I64 => I64, } } + +/// TODO(10248) This is removed in the next stack switching PR. It stops the +/// compiler from complaining about the stack switching libcalls being dead +/// code. +#[allow( + dead_code, + reason = "Dummy function to supress more dead code warnings" +)] +pub fn use_stack_switching_libcalls() { + let _ = BuiltinFunctions::delete_me_print_str; + let _ = BuiltinFunctions::delete_me_print_int; + let _ = BuiltinFunctions::delete_me_print_pointer; + + let _ = BuiltinFunctions::cont_new; + let _ = BuiltinFunctions::table_grow_cont_obj; + let _ = BuiltinFunctions::table_fill_cont_obj; +} diff --git a/crates/cranelift/src/gc/enabled.rs b/crates/cranelift/src/gc/enabled.rs index 04ae2ff9b7d8..743ea93af3e3 100644 --- a/crates/cranelift/src/gc/enabled.rs +++ b/crates/cranelift/src/gc/enabled.rs @@ -148,7 +148,12 @@ fn read_field_at_addr( .call(get_interned_func_ref, &[vmctx, func_ref_id, expected_ty]); builder.func.dfg.first_result(call_inst) } - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => { + // TODO(#10248) GC integration for stack switching + return Err(wasmtime_environ::WasmError::Unsupported( + "Stack switching feature not compatbile with GC, yet".to_string(), + )); + } }, }, }; @@ -1011,6 +1016,8 @@ pub fn translate_ref_test( | WasmHeapType::NoExtern | WasmHeapType::Func | WasmHeapType::NoFunc + | WasmHeapType::Cont + | WasmHeapType::NoCont | WasmHeapType::I31 => unreachable!("handled top, bottom, and i31 types above"), // For these abstract but non-top and non-bottom types, we check the @@ -1063,8 +1070,12 @@ pub fn translate_ref_test( func_env.is_subtype(builder, actual_shared_ty, expected_shared_ty) } - - WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapType::ConcreteCont(_) => { + // TODO(#10248) GC integration for stack switching + return Err(wasmtime_environ::WasmError::Unsupported( + "Stack switching feature not compatbile with GC, yet".to_string(), + )); + } }; builder.ins().jump(continue_block, &[result]); @@ -1403,8 +1414,9 @@ impl FuncEnvironment<'_> { WasmHeapType::Func | WasmHeapType::ConcreteFunc(_) | WasmHeapType::NoFunc => { unreachable!() } - - WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => { + unreachable!() + } }; match (ty.nullable, might_be_i31) { diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index a98a9c04b77d..8393ae3dc124 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -61,6 +61,12 @@ pub const TRAP_HEAP_MISALIGNED: TrapCode = TrapCode::unwrap_user(Trap::HeapMisaligned as u8 + TRAP_OFFSET); pub const TRAP_TABLE_OUT_OF_BOUNDS: TrapCode = TrapCode::unwrap_user(Trap::TableOutOfBounds as u8 + TRAP_OFFSET); +pub const TRAP_UNHANDLED_TAG: TrapCode = + TrapCode::unwrap_user(Trap::UnhandledTag as u8 + TRAP_OFFSET); +pub const TRAP_CONTINUATION_ALREADY_CONSUMED: TrapCode = + TrapCode::unwrap_user(Trap::ContinuationAlreadyConsumed as u8 + TRAP_OFFSET); +pub const TRAP_DELETE_ME_DEBUG_ASSERTION: TrapCode = + TrapCode::unwrap_user(Trap::DeleteMeDebugAssertion as u8 + TRAP_OFFSET); pub const TRAP_CAST_FAILURE: TrapCode = TrapCode::unwrap_user(Trap::CastFailure as u8 + TRAP_OFFSET); @@ -202,7 +208,11 @@ fn reference_type(wasm_ht: WasmHeapType, pointer_type: ir::Type) -> ir::Type { match wasm_ht.top() { WasmHeapTopType::Func => pointer_type, WasmHeapTopType::Any | WasmHeapTopType::Extern => ir::types::I32, - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => + // TODO(10248) This is added in a follow-up PR + { + unimplemented!("codegen for stack switching types not implemented, yet") + } } } diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index 8bb55974ec56..4a30e262adff 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -2876,6 +2876,56 @@ pub fn translate_operator( // representation, so we don't actually need to do anything. } + Operator::ContNew { cont_type_index: _ } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::ContBind { + argument_index: _, + result_index: _, + } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::Suspend { tag_index: _ } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::Resume { + cont_type_index: _, + resume_table: _, + } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::ResumeThrow { + cont_type_index: _, + tag_index: _, + resume_table: _, + } => { + // TODO(10248) This depends on exception handling + return Err(wasmtime_environ::WasmError::Unsupported( + "resume.throw instructions not supported, yet".to_string(), + )); + } + Operator::Switch { + cont_type_index: _, + tag_index: _, + } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::GlobalAtomicGet { .. } | Operator::GlobalAtomicSet { .. } | Operator::GlobalAtomicRmwAdd { .. } @@ -2917,17 +2967,6 @@ pub fn translate_operator( )); } - Operator::ContNew { .. } - | Operator::ContBind { .. } - | Operator::Suspend { .. } - | Operator::Resume { .. } - | Operator::ResumeThrow { .. } - | Operator::Switch { .. } => { - return Err(wasm_unsupported!( - "stack-switching operators are not yet implemented" - )); - } - Operator::I64MulWideS => { let (arg1, arg2) = state.pop2(); let arg1 = builder.ins().sextend(I128, arg1); diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index a446829af7d8..1c70dd964cd7 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -206,6 +206,32 @@ macro_rules! foreach_builtin_function { // Raises an unconditional trap where the trap information must have // been previously filled in. raise(vmctx: vmctx); + + // Creates a new continuation from a funcref. + cont_new(vmctx: vmctx, r: pointer, param_count: u32, result_count: u32) -> pointer; + + // FIXME(frank-emrich) The next three builtins are used by the debug printing mechanism. + // They are not supposed to be part of the final upstreamed code. + // + // Prints a 'static str, represented as a + // pointer and a length. + delete_me_print_str(vmctx: vmctx, s: pointer, len : u64); + // Prints integer + delete_me_print_int(vmctx: vmctx, arg : u64); + // Prints pointer, formatted as hex. + delete_me_print_pointer(vmctx: vmctx, arg : pointer); + + // Returns an index for Wasm's `table.grow` instruction + // for `contobj`s. Note that the initial + // Option (i.e., the value to fill the new + // slots with) is split into two arguments: The underlying + // continuation reference and the revision count. To + // denote the continuation being `None`, `init_contref` + // may be 0. + table_grow_cont_obj(vmctx: vmctx, table: u32, delta: u64, init_contref: pointer, init_revision: u64) -> pointer; + // `value_contref` and `value_revision` together encode + // the Option, as in previous libcall. + table_fill_cont_obj(vmctx: vmctx, table: u32, dst: u64, value_contref: pointer, value_revision: u64, len: u64) -> bool; } }; } @@ -347,6 +373,7 @@ impl BuiltinFunctionIndex { (@get memory32_grow pointer) => (TrapSentinel::NegativeTwo); (@get table_grow_func_ref pointer) => (TrapSentinel::NegativeTwo); (@get table_grow_gc_ref pointer) => (TrapSentinel::NegativeTwo); + (@get table_grow_cont_obj pointer) => (TrapSentinel::NegativeTwo); // Atomics-related functions return a negative value indicating trap // indicate a trap. @@ -371,6 +398,8 @@ impl BuiltinFunctionIndex { (@get intern_func_ref_for_gc_heap u64) => (return None); (@get is_subtype u32) => (return None); + (@get cont_new pointer) => (TrapSentinel::Negative); + // Bool-returning functions use `false` as an indicator of a trap. (@get $name:ident bool) => (TrapSentinel::Falsy); diff --git a/crates/environ/src/gc.rs b/crates/environ/src/gc.rs index 4fa7d8745bab..7e2215b354fc 100644 --- a/crates/environ/src/gc.rs +++ b/crates/environ/src/gc.rs @@ -40,10 +40,15 @@ pub const VM_GC_HEADER_TYPE_INDEX_OFFSET: u32 = 4; /// Get the byte size of the given Wasm type when it is stored inside the GC /// heap. pub fn byte_size_of_wasm_ty_in_gc_heap(ty: &WasmStorageType) -> u32 { + use crate::{WasmHeapType::*, WasmRefType}; match ty { WasmStorageType::I8 => 1, WasmStorageType::I16 => 2, WasmStorageType::Val(ty) => match ty { + WasmValType::Ref(WasmRefType { + nullable: _, + heap_type: ConcreteCont(_) | Cont, + }) => unimplemented!("Stack switching feature not compatbile with GC, yet"), WasmValType::I32 | WasmValType::F32 | WasmValType::Ref(_) => 4, WasmValType::I64 | WasmValType::F64 => 8, WasmValType::V128 => 16, @@ -162,7 +167,9 @@ pub trait GcTypeLayouts { WasmCompositeInnerType::Array(ty) => Some(self.array_layout(ty).into()), WasmCompositeInnerType::Struct(ty) => Some(self.struct_layout(ty).into()), WasmCompositeInnerType::Func(_) => None, - WasmCompositeInnerType::Cont(_) => None, + WasmCompositeInnerType::Cont(_) => { + unimplemented!("Stack switching feature not compatbile with GC, yet") + } } } diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index 511b4a73cfec..3b42869f58b8 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -29,6 +29,7 @@ pub mod obj; mod ref_bits; mod scopevec; mod stack_map; +pub mod stack_switching; mod trap_encoding; mod tunables; mod types; diff --git a/crates/environ/src/stack_switching.rs b/crates/environ/src/stack_switching.rs new file mode 100644 index 000000000000..6a781faea905 --- /dev/null +++ b/crates/environ/src/stack_switching.rs @@ -0,0 +1,78 @@ +//! This module contains basic type definitions used by the implementation of +//! the stack switching proposal. + +/// FIXME(frank-emrich) Will remove in the final upstreamed version +#[allow(dead_code, reason = "Only accessed in debug builds")] +pub const ENABLE_DEBUG_PRINTING: bool = false; + +/// FIXME(frank-emrich) Will remove in the final upstreamed version +#[macro_export] +macro_rules! debug_println { + ($( $args:expr ),+ ) => { + #[cfg(debug_assertions)] + if ENABLE_DEBUG_PRINTING { + #[cfg(feature = "std")] + println!($($args),*); + } + } +} + +/// Runtime configuration options for stack switching that can be set +/// via the command line. +/// +/// Part of wasmtime::config::Config type (which is not in scope in this crate). +#[derive(Debug, Clone)] +pub struct StackSwitchingConfig { + /// The (fixed) size of a continuation stack. + pub stack_size: usize, +} + +impl Default for StackSwitchingConfig { + fn default() -> Self { + /// Default size for continuation stacks + const DEFAULT_FIBER_SIZE: usize = 2097152; // 2MB = 512 pages of 4k + + Self { + stack_size: DEFAULT_FIBER_SIZE, + } + } +} + +/// Discriminant of variant `Absent` in +/// `wasmtime::runtime::vm::stack_switching::VMStackChain`. +pub const STACK_CHAIN_ABSENT_DISCRIMINANT: usize = 0; +/// Discriminant of variant `InitialStack` in +/// `wasmtime::runtime::vm::stack_switching::VMStackChain`. +pub const STACK_CHAIN_INITIAL_STACK_DISCRIMINANT: usize = 1; +/// Discriminant of variant `Continiation` in +/// `wasmtime::runtime::vm::stack_switching::VMStackChain`. +pub const STACK_CHAIN_CONTINUATION_DISCRIMINANT: usize = 2; + +/// Discriminant of variant `Fresh` in +/// `runtime::vm::stack_switching::VMStackState`. +pub const STACK_STATE_FRESH_DISCRIMINANT: u32 = 0; +/// Discriminant of variant `Running` in +/// `runtime::vm::stack_switching::VMStackState`. +pub const STACK_STATE_RUNNING_DISCRIMINANT: u32 = 1; +/// Discriminant of variant `Parent` in +/// `runtime::vm::stack_switching::VMStackState`. +pub const STACK_STATE_PARENT_DISCRIMINANT: u32 = 2; +/// Discriminant of variant `Suspended` in +/// `runtime::vm::stack_switching::VMStackState`. +pub const STACK_STATE_SUSPENDED_DISCRIMINANT: u32 = 3; +/// Discriminant of variant `Returned` in +/// `runtime::vm::stack_switching::VMStackState`. +pub const STACK_STATE_RETURNED_DISCRIMINANT: u32 = 4; + +/// Discriminant of variant `Return` in +/// `runtime::vm::stack_switching::ControlEffect`. +pub const CONTROL_EFFECT_RETURN_DISCRIMINANT: u32 = 0; +/// Discriminant of variant `Resume` in +/// `runtime::vm::stack_switching::ControlEffect`. +pub const CONTROL_EFFECT_RESUME_DISCRIMINANT: u32 = 1; +/// Discriminant of variant `Suspend` in +/// `runtime::vm::stack_switching::ControlEffect`. +pub const CONTROL_EFFECT_SUSPEND_DISCRIMINANT: u32 = 2; +/// Discriminant of variant `Switch` in +/// `runtime::vm::stack_switching::ControlEffect`. +pub const CONTROL_EFFECT_SWITCH_DISCRIMINANT: u32 = 3; diff --git a/crates/environ/src/trap_encoding.rs b/crates/environ/src/trap_encoding.rs index 38e5e5dd4119..b1ed2885110a 100644 --- a/crates/environ/src/trap_encoding.rs +++ b/crates/environ/src/trap_encoding.rs @@ -92,6 +92,16 @@ pub enum Trap { /// Async-lifted export failed to produce a result by calling `task.return` /// before returning `STATUS_DONE` and/or after all host tasks completed. NoAsyncResult, + + /// We are suspending to a tag for which there is no active handler. + UnhandledTag, + + /// Attempt to resume a continuation twice. + ContinuationAlreadyConsumed, + + /// FIXME(frank-emrich) Only used for stack switching debugging code, to be + /// removed from final upstreamed code. + DeleteMeDebugAssertion, // if adding a variant here be sure to update the `check!` macro below } @@ -129,6 +139,9 @@ impl Trap { CastFailure CannotEnterComponent NoAsyncResult + UnhandledTag + ContinuationAlreadyConsumed + DeleteMeDebugAssertion } None @@ -160,6 +173,9 @@ impl fmt::Display for Trap { CastFailure => "cast failure", CannotEnterComponent => "cannot enter component instance", NoAsyncResult => "async-lifted export failed to produce a result", + UnhandledTag => "unhandled tag", + ContinuationAlreadyConsumed => "continuation already consumed", + DeleteMeDebugAssertion => "triggered debug assertion", }; write!(f, "wasm trap: {desc}") } diff --git a/crates/environ/src/types.rs b/crates/environ/src/types.rs index 14829377ea43..b9e4056c6f7a 100644 --- a/crates/environ/src/types.rs +++ b/crates/environ/src/types.rs @@ -232,6 +232,14 @@ impl WasmValType { size => panic!("invalid int bits for WasmValType: {size}"), } } + + /// TODO + pub fn unwrap_ref_type(&self) -> WasmRefType { + match self { + WasmValType::Ref(ref_type) => *ref_type, + _ => panic!("Called WasmValType::unwrap_ref_type on non-reference type"), + } + } } /// WebAssembly reference type -- equivalent of `wasmparser`'s RefType @@ -801,6 +809,15 @@ impl WasmContType { pub fn new(idx: EngineOrModuleTypeIndex) -> Self { WasmContType(idx) } + + /// Returns the (module interned) index to the underlying function type. + pub fn unwrap_interned_type_index(self) -> ModuleInternedTypeIndex { + match self.0 { + EngineOrModuleTypeIndex::Engine(_) => panic!("not module interned"), + EngineOrModuleTypeIndex::Module(idx) => idx, + EngineOrModuleTypeIndex::RecGroup(_) => todo!(), + } + } } impl TypeTrace for WasmContType { @@ -2217,12 +2234,11 @@ pub trait TypeConvert { wasmparser::AbstractHeapType::Struct => WasmHeapType::Struct, wasmparser::AbstractHeapType::None => WasmHeapType::None, - wasmparser::AbstractHeapType::Exn - | wasmparser::AbstractHeapType::NoExn - | wasmparser::AbstractHeapType::Cont - | wasmparser::AbstractHeapType::NoCont => { + wasmparser::AbstractHeapType::Exn | wasmparser::AbstractHeapType::NoExn => { unimplemented!("unsupported heap type {ty:?}"); } + wasmparser::AbstractHeapType::Cont => WasmHeapType::Cont, + wasmparser::AbstractHeapType::NoCont => WasmHeapType::NoCont, }, _ => unimplemented!("unsupported heap type {ty:?}"), } diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 01c283317126..b8e03a3996e2 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -164,6 +164,12 @@ pub trait PtrSize { 4 } + /// This is the size of the largest value type (i.e. a V128). + #[inline] + fn maximum_value_size(&self) -> u8 { + self.size_of_vmglobal_definition() + } + // Offsets within `VMStoreContext` /// Return the offset of the `fuel_consumed` field of `VMStoreContext` @@ -199,6 +205,11 @@ pub trait PtrSize { self.vmstore_context_last_wasm_exit_pc() + self.size() } + /// Return the offset of the `stack_chain` field of `VMStoreContext`. + fn vmstore_context_stack_chain(&self) -> u8 { + self.vmstore_context_last_wasm_entry_fp() + self.size() + } + // Offsets within `VMMemoryDefinition` /// The offset of the `base` field. @@ -236,6 +247,122 @@ pub trait PtrSize { .unwrap() } + /// Return the size of `VMStackChain`. + fn size_of_vmstack_chain(&self) -> u8 { + 2 * self.size() + } + + // Offsets within `VMStackLimits` + + /// Return the offset of `VMStackLimits::stack_limit`. + fn vmstack_limits_stack_limit(&self) -> u8 { + 0 + } + + /// Return the offset of `VMStackLimits::last_wasm_entry_fp`. + fn vmstack_limits_last_wasm_entry_fp(&self) -> u8 { + self.size() + } + + // Offsets within `VMArray` + + /// Return the offset of `VMArray::length`. + fn vmarray_length(&self) -> u8 { + 0 + } + + /// Return the offset of `VMArray::capacity`. + fn vmarray_capacity(&self) -> u8 { + 4 + } + + /// Return the offset of `VMArray::data`. + fn vmarray_data(&self) -> u8 { + 8 + } + + /// Return the size of `VMArray`. + fn size_of_vmarray(&self) -> u8 { + 8 + self.size() + } + + // Offsets within `VMCommonStackInformation` + + /// Return the offset of `VMCommonStackInformation::limits`. + fn vmcommon_stack_information_limits(&self) -> u8 { + 0 * self.size() + } + + /// Return the offset of `VMCommonStackInformation::state`. + fn vmcommon_stack_information_state(&self) -> u8 { + 2 * self.size() + } + + /// Return the offset of `VMCommonStackInformation::handlers`. + fn vmcommon_stack_information_handlers(&self) -> u8 { + u8::try_from(align( + self.vmcommon_stack_information_state() as u32 + 4, + u32::from(self.size()), + )) + .unwrap() + } + + /// Return the offset of `VMCommonStackInformation::first_switch_handler_index`. + fn vmcommon_stack_information_first_switch_handler_index(&self) -> u8 { + self.vmcommon_stack_information_handlers() + self.size_of_vmarray() + } + + /// Return the size of `VMCommonStackInformation`. + fn size_of_vmcommon_stack_information(&self) -> u8 { + u8::try_from(align( + self.vmcommon_stack_information_first_switch_handler_index() as u32 + 4, + u32::from(self.size()), + )) + .unwrap() + } + + // Offsets within `VMContRef` + + /// Return the offset of `VMContRef::common_stack_information`. + fn vmcontref_common_stack_information(&self) -> u8 { + 0 * self.size() + } + + /// Return the offset of `VMContRef::parent_chain`. + fn vmcontref_parent_chain(&self) -> u8 { + u8::try_from(align( + (self.vmcontref_common_stack_information() + self.size_of_vmcommon_stack_information()) + as u32, + u32::from(self.size()), + )) + .unwrap() + } + + /// Return the offset of `VMContRef::last_ancestor`. + fn vmcontref_last_ancestor(&self) -> u8 { + self.vmcontref_parent_chain() + 2 * self.size() + } + + /// Return the offset of `VMContRef::revision`. + fn vmcontref_revision(&self) -> u8 { + self.vmcontref_last_ancestor() + self.size() + } + + /// Return the offset of `VMContRef::stack`. + fn vmcontref_stack(&self) -> u8 { + self.vmcontref_revision() + 8 + } + + /// Return the offset of `VMContRef::args`. + fn vmcontref_args(&self) -> u8 { + self.vmcontref_stack() + 3 * self.size() + } + + /// Return the offset of `VMContRef::values`. + fn vmcontref_values(&self) -> u8 { + self.vmcontref_args() + self.size_of_vmarray() + } + /// Return the offset to the `magic` value in this `VMContext`. #[inline] fn vmctx_magic(&self) -> u8 { diff --git a/crates/fuzzing/src/generators/config.rs b/crates/fuzzing/src/generators/config.rs index 5a484a3de661..732e450e602d 100644 --- a/crates/fuzzing/src/generators/config.rs +++ b/crates/fuzzing/src/generators/config.rs @@ -144,6 +144,7 @@ impl Config { hogs_memory: _, nan_canonicalization: _, gc_types: _, + stack_switching: _, } = test.config; // Enable/disable some proposals that aren't configurable in wasm-smith diff --git a/crates/misc/component-test-util/src/lib.rs b/crates/misc/component-test-util/src/lib.rs index d300f02e7577..8b6bfeed0692 100644 --- a/crates/misc/component-test-util/src/lib.rs +++ b/crates/misc/component-test-util/src/lib.rs @@ -168,6 +168,7 @@ pub fn apply_test_config(config: &mut Config, test_config: &wasmtime_wast_util:: component_model_async, nan_canonicalization, simd, + stack_switching, hogs_memory: _, gc_types: _, @@ -192,7 +193,8 @@ pub fn apply_test_config(config: &mut Config, test_config: &wasmtime_wast_util:: // To avoid needing to enable all of them at once implicitly enable // downstream proposals once the end proposal is enabled (e.g. when enabling // gc that also enables function-references and reference-types). - let function_references = gc || function_references.unwrap_or(false); + let stack_switching = stack_switching.unwrap_or(false); + let function_references = gc || stack_switching || function_references.unwrap_or(false); let reference_types = function_references || reference_types.unwrap_or(false); let simd = relaxed_simd || simd.unwrap_or(false); @@ -210,5 +212,6 @@ pub fn apply_test_config(config: &mut Config, test_config: &wasmtime_wast_util:: .wasm_extended_const(extended_const) .wasm_wide_arithmetic(wide_arithmetic) .wasm_component_model_async(component_model_async) + .wasm_stack_switching(stack_switching) .cranelift_nan_canonicalization(nan_canonicalization); } diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 67ff4001f77b..832f29d98101 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -93,7 +93,7 @@ memfd = { workspace = true, optional = true } mach2 = { workspace = true, optional = true } [target.'cfg(unix)'.dependencies] -rustix = { workspace = true, optional = true } +rustix = { workspace = true, optional = true, features = ["mm", "param"] } [target.'cfg(target_arch = "s390x")'.dependencies] psm = { workspace = true, optional = true } @@ -143,6 +143,7 @@ default = [ 'runtime', 'component-model', 'threads', + 'stack-switching', 'std', ] @@ -308,6 +309,11 @@ threads = [ "std", ] +stack-switching = [ + "std", + "runtime" +] + # Controls whether backtraces will attempt to parse DWARF information in # WebAssembly modules and components to provide filenames and line numbers in # stack traces. diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 75c1078c127a..73059cc61bcc 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -8,7 +8,9 @@ use std::path::Path; use wasmparser::WasmFeatures; #[cfg(feature = "cache")] use wasmtime_cache::CacheConfig; -use wasmtime_environ::{ConfigTunables, TripleExt, Tunables}; +use wasmtime_environ::{ + stack_switching::StackSwitchingConfig, ConfigTunables, TripleExt, Tunables, +}; #[cfg(feature = "runtime")] use crate::memory::MemoryCreator; @@ -127,6 +129,14 @@ pub struct Config { profiling_strategy: ProfilingStrategy, tunables: ConfigTunables, + /// Runtime configuration for the stack switching feature. + /// The structure is defined in the + /// `wasmtime_environ::stack_switching` module, so that we can + /// hand out the configuration object in the interface of + /// `wasmtime_runtime::Store` trait, where the full `Config` type + /// is not in scope. + pub(crate) stack_switching_config: StackSwitchingConfig, + #[cfg(feature = "cache")] pub(crate) cache_config: CacheConfig, #[cfg(feature = "runtime")] @@ -227,6 +237,7 @@ impl Config { tunables: ConfigTunables::default(), #[cfg(any(feature = "cranelift", feature = "winch"))] compiler_config: CompilerConfig::default(), + stack_switching_config: StackSwitchingConfig::default(), target: None, #[cfg(feature = "gc")] collector: Collector::default(), @@ -777,6 +788,12 @@ impl Config { self } + /// Configures the size of the stacks created with cont.new instructions. + pub fn stack_switching_stack_size(&mut self, size: usize) -> &mut Self { + self.stack_switching_config.stack_size = size; + self + } + fn wasm_feature(&mut self, flag: WasmFeatures, enable: bool) -> &mut Self { self.enabled_features.set(flag, enable); self.disabled_features.set(flag, !enable); @@ -2024,18 +2041,37 @@ impl Config { // `threads` proposal, notably shared memory, because Rust can't // safely implement loads/stores in the face of shared memory. if self.compiler_target().is_pulley() { - return WasmFeatures::THREADS; + return WasmFeatures::THREADS | WasmFeatures::STACK_SWITCHING; } - // Other Cranelift backends are either 100% missing or complete - // at this time, so no need to further filter. - WasmFeatures::empty() + use target_lexicon::*; + match self.compiler_target() { + Triple { + architecture: Architecture::X86_64 | Architecture::X86_64h, + operating_system: + OperatingSystem::Linux + | OperatingSystem::MacOSX(_) + | OperatingSystem::Darwin(_), + .. + } => { + // Other Cranelift backends are either 100% missing or complete + // at this time, so no need to further filter. + WasmFeatures::empty() + } + + _ => { + // On platforms other than x64 Unix-like, we don't + // support stack switching. + WasmFeatures::STACK_SWITCHING + } + } } Some(Strategy::Winch) => { let mut unsupported = WasmFeatures::GC | WasmFeatures::FUNCTION_REFERENCES | WasmFeatures::RELAXED_SIMD | WasmFeatures::TAIL_CALL + | WasmFeatures::STACK_SWITCHING | WasmFeatures::GC_TYPES; match self.compiler_target().architecture { target_lexicon::Architecture::Aarch64(_) => { @@ -2426,6 +2462,27 @@ impl Config { bail!("cannot disable the simd proposal but enable the relaxed simd proposal"); } + if features.contains(WasmFeatures::STACK_SWITCHING) { + use target_lexicon::OperatingSystem; + let model = match target.operating_system { + OperatingSystem::Windows => "update_windows_tib", + OperatingSystem::Linux + | OperatingSystem::MacOSX(_) + | OperatingSystem::Darwin(_) => "basic", + _ => bail!("stack-switching feature not supported on this platform "), + }; + + if !self + .compiler_config + .ensure_setting_unset_or_given("stack_switch_model".into(), model.into()) + { + bail!( + "compiler option 'stack_switch_model' must be set to '{}' on this platform", + model + ); + } + } + // Apply compiler settings and flags for (k, v) in self.compiler_config.settings.iter() { compiler.set(k, v)?; diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index 532c08a98382..473282265e63 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -377,6 +377,24 @@ impl Engine { } } + // stack switch model must match the current OS + "stack_switch_model" => { + if self.features().contains(WasmFeatures::STACK_SWITCHING) { + use target_lexicon::OperatingSystem; + let expected = + match target.operating_system { + OperatingSystem::Windows => "update_windows_tib", + OperatingSystem::Linux + | OperatingSystem::MacOSX(_) + | OperatingSystem::Darwin(_) => "basic", + _ => { return Err(String::from("stack-switching feature not supported on this platform")); } + }; + *value == FlagValue::Enum(expected) + } else { + return Ok(()) + } + } + // These settings don't affect the interface or functionality of // the module itself, so their configuration values shouldn't // matter. @@ -394,7 +412,6 @@ impl Engine { | "bb_padding_log2_minus_one" | "machine_code_cfg_info" | "tls_model" // wasmtime doesn't use tls right now - | "stack_switch_model" // wasmtime doesn't use stack switching right now | "opt_level" // opt level doesn't change semantics | "enable_alias_analysis" // alias analysis-based opts don't change semantics | "probestack_size_log2" // probestack above asserted disabled diff --git a/crates/wasmtime/src/runtime/externals/global.rs b/crates/wasmtime/src/runtime/externals/global.rs index 02a9669c20f4..3bca5dd0f12b 100644 --- a/crates/wasmtime/src/runtime/externals/global.rs +++ b/crates/wasmtime/src/runtime/externals/global.rs @@ -130,6 +130,10 @@ impl Global { }) .into(), ), + HeapType::NoCont | HeapType::ConcreteCont(_) | HeapType::Cont => { + // TODO(#10248) Required to support stack switching in the embedder API. + unimplemented!() + } HeapType::NoExtern => Ref::Extern(None), diff --git a/crates/wasmtime/src/runtime/externals/table.rs b/crates/wasmtime/src/runtime/externals/table.rs index 40fd1056d6d2..19ee59340efb 100644 --- a/crates/wasmtime/src/runtime/externals/table.rs +++ b/crates/wasmtime/src/runtime/externals/table.rs @@ -188,6 +188,11 @@ impl Table { ty => unreachable!("not a top type: {ty:?}"), } } + + runtime::TableElement::ContRef(_c) => { + // TODO(#10248) Required to support stack switching in the embedder API. + unimplemented!() + } } } } diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs index e75484b36cdf..868d02690e66 100644 --- a/crates/wasmtime/src/runtime/func.rs +++ b/crates/wasmtime/src/runtime/func.rs @@ -1,7 +1,8 @@ use crate::prelude::*; +use crate::runtime::vm::stack_switching::VMCommonStackInformation; use crate::runtime::vm::{ ExportFunction, InterpreterRef, SendSyncPtr, StoreBox, VMArrayCallHostFuncContext, VMContext, - VMFuncRef, VMFunctionImport, VMOpaqueContext, + VMFuncRef, VMFunctionImport, VMOpaqueContext, VMStoreContext, }; use crate::runtime::Uninhabited; use crate::store::{AutoAssertNoGc, StoreData, StoreOpaque, Stored}; @@ -357,6 +358,7 @@ macro_rules! for_each_function_signature { } mod typed; +use crate::runtime::vm::stack_switching::VMStackChain; pub use typed::*; impl Func { @@ -1191,6 +1193,7 @@ impl Func { results.len() ); } + for (ty, arg) in ty.params().zip(params) { arg.ensure_matches_ty(opaque, &ty) .context("argument type mismatch")?; @@ -1597,103 +1600,173 @@ pub(crate) fn invoke_wasm_and_catch_traps( closure: impl FnMut(NonNull, Option>) -> bool, ) -> Result<()> { unsafe { - let exit = enter_wasm(store); + // The `enter_wasm` call below will reset the store's `stack_chain` to + // a new `InitialStack`, pointing to the stack-allocated + // `initial_stack_csi`. + let mut initial_stack_csi = VMCommonStackInformation::running_default(); + // Stores some state of the runtime just before entering Wasm. Will be + // restored upon exiting Wasm. Note that the `CallThreadState` that is + // created by the `catch_traps` call below will store a pointer to this + // stack-allocated `previous_runtime_state`. + let previous_runtime_state = EntryStoreContext::enter_wasm(store, &mut initial_stack_csi); if let Err(trap) = store.0.call_hook(CallHook::CallingWasm) { - exit_wasm(store, exit); return Err(trap); } - let result = crate::runtime::vm::catch_traps(store, closure); - exit_wasm(store, exit); + let result = crate::runtime::vm::catch_traps(store, &previous_runtime_state, closure); + core::mem::drop(previous_runtime_state); store.0.call_hook(CallHook::ReturningFromWasm)?; result.map_err(|t| crate::trap::from_runtime_box(store.0, t)) } } -/// This function is called to register state within `Store` whenever -/// WebAssembly is entered within the `Store`. -/// -/// This function sets up various limits such as: -/// -/// * The stack limit. This is what ensures that we limit the stack space -/// allocated by WebAssembly code and it's relative to the initial stack -/// pointer that called into wasm. -/// -/// This function may fail if the stack limit can't be set because an -/// interrupt already happened. -fn enter_wasm(store: &mut StoreContextMut<'_, T>) -> Option { - // If this is a recursive call, e.g. our stack limit is already set, then - // we may be able to skip this function. - // - // For synchronous stores there's nothing else to do because all wasm calls - // happen synchronously and on the same stack. This means that the previous - // stack limit will suffice for the next recursive call. - // - // For asynchronous stores then each call happens on a separate native - // stack. This means that the previous stack limit is no longer relevant - // because we're on a separate stack. - if unsafe { *store.0.vm_store_context().stack_limit.get() } != usize::MAX - && !store.0.async_support() - { - return None; - } - - // Ignore this stack pointer business on miri since we can't execute wasm - // anyway and the concept of a stack pointer on miri is a bit nebulous - // regardless. - if cfg!(miri) { - return None; - } - - // When Cranelift has support for the host then we might be running native - // compiled code meaning we need to read the actual stack pointer. If - // Cranelift can't be used though then we're guaranteed to be running pulley - // in which case this stack pointer isn't actually used as Pulley has custom - // mechanisms for stack overflow. - #[cfg(has_host_compiler_backend)] - let stack_pointer = crate::runtime::vm::get_stack_pointer(); - #[cfg(not(has_host_compiler_backend))] - let stack_pointer = { - use wasmtime_environ::TripleExt; - debug_assert!(store.engine().target().is_pulley()); - usize::MAX - }; +/// This type helps managing the state of the runtime when entering and exiting +/// Wasm. To this end, it contains a subset of the data in `VMStoreContext`.. +/// Upon entering Wasm, it updates various runtime fields and their +/// original values saved in this struct. Upon exiting Wasm, the previous values +/// are restored. +// FIXME(frank-emrich) Do the fields in here need to be (Unsafe)Cells? +pub struct EntryStoreContext { + /// If set, contains value of `stack_limit` field to restore in + /// `VMRuntimeLimits` when exiting Wasm. + pub stack_limit: Option, + /// Contains value of `last_wasm_exit_pc` field to restore in + /// `VMRuntimeLimits` when exiting Wasm. + pub last_wasm_exit_pc: usize, + /// Contains value of `last_wasm_exit_fp` field to restore in + /// `VMRuntimeLimits` when exiting Wasm. + pub last_wasm_exit_fp: usize, + /// Contains value of `last_wasm_entry_fp` field to restore in + /// `VMRuntimeLimits` when exiting Wasm. + pub last_wasm_entry_fp: usize, + /// Contains value of `stack_chain` field to restore in + /// `Store` when exiting Wasm. + pub stack_chain: VMStackChain, + + /// We need a pointer to the runtime limits, so we can update them from + /// `drop`/`exit_wasm`. + vm_store_context: *const VMStoreContext, +} - // Determine the stack pointer where, after which, any wasm code will - // immediately trap. This is checked on the entry to all wasm functions. - // - // Note that this isn't 100% precise. We are requested to give wasm - // `max_wasm_stack` bytes, but what we're actually doing is giving wasm - // probably a little less than `max_wasm_stack` because we're - // calculating the limit relative to this function's approximate stack - // pointer. Wasm will be executed on a frame beneath this one (or next - // to it). In any case it's expected to be at most a few hundred bytes - // of slop one way or another. When wasm is typically given a MB or so - // (a million bytes) the slop shouldn't matter too much. - // - // After we've got the stack limit then we store it into the `stack_limit` - // variable. - let wasm_stack_limit = stack_pointer - store.engine().config().max_wasm_stack; - let prev_stack = unsafe { - mem::replace( - &mut *store.0.vm_store_context().stack_limit.get(), - wasm_stack_limit, - ) - }; +impl EntryStoreContext { + /// This function is called to update and save state when + /// WebAssembly is entered within the `Store`. + /// + /// This updates various fields such as: + /// + /// * The stack limit. This is what ensures that we limit the stack space + /// allocated by WebAssembly code and it's relative to the initial stack + /// pointer that called into wasm. + /// + /// It also saves the different last_wasm_* values in the `VMRuntimeLimits`. + pub fn enter_wasm( + store: &mut StoreContextMut<'_, T>, + initial_stack_information: *mut VMCommonStackInformation, + ) -> Self { + let stack_limit; - Some(prev_stack) -} + // If this is a recursive call, e.g. our stack limit is already set, then + // we may be able to skip this function. + // + // For synchronous stores there's nothing else to do because all wasm calls + // happen synchronously and on the same stack. This means that the previous + // stack limit will suffice for the next recursive call. + // + // For asynchronous stores then each call happens on a separate native + // stack. This means that the previous stack limit is no longer relevant + // because we're on a separate stack. + if unsafe { *store.0.vm_store_context().stack_limit.get() } != usize::MAX + && !store.0.async_support() + { + stack_limit = None; + } + // Ignore this stack pointer business on miri since we can't execute wasm + // anyway and the concept of a stack pointer on miri is a bit nebulous + // regardless. + else if cfg!(miri) { + stack_limit = None; + } else { + // When Cranelift has support for the host then we might be running native + // compiled code meaning we need to read the actual stack pointer. If + // Cranelift can't be used though then we're guaranteed to be running pulley + // in which case this stack pointer isn't actually used as Pulley has custom + // mechanisms for stack overflow. + #[cfg(has_host_compiler_backend)] + let stack_pointer = crate::runtime::vm::get_stack_pointer(); + #[cfg(not(has_host_compiler_backend))] + let stack_pointer = { + use wasmtime_environ::TripleExt; + debug_assert!(store.engine().target().is_pulley()); + usize::MAX + }; -fn exit_wasm(store: &mut StoreContextMut<'_, T>, prev_stack: Option) { - // If we don't have a previous stack pointer to restore, then there's no - // cleanup we need to perform here. - let prev_stack = match prev_stack { - Some(stack) => stack, - None => return, - }; + // Determine the stack pointer where, after which, any wasm code will + // immediately trap. This is checked on the entry to all wasm functions. + // + // Note that this isn't 100% precise. We are requested to give wasm + // `max_wasm_stack` bytes, but what we're actually doing is giving wasm + // probably a little less than `max_wasm_stack` because we're + // calculating the limit relative to this function's approximate stack + // pointer. Wasm will be executed on a frame beneath this one (or next + // to it). In any case it's expected to be at most a few hundred bytes + // of slop one way or another. When wasm is typically given a MB or so + // (a million bytes) the slop shouldn't matter too much. + // + // After we've got the stack limit then we store it into the `stack_limit` + // variable. + let wasm_stack_limit = stack_pointer - store.engine().config().max_wasm_stack; + let prev_stack = unsafe { + mem::replace( + &mut *store.0.vm_store_context().stack_limit.get(), + wasm_stack_limit, + ) + }; + stack_limit = Some(prev_stack); + } - unsafe { - *store.0.vm_store_context().stack_limit.get() = prev_stack; + unsafe { + let last_wasm_exit_pc = *store.0.vm_store_context().last_wasm_exit_pc.get(); + let last_wasm_exit_fp = *store.0.vm_store_context().last_wasm_exit_fp.get(); + let last_wasm_entry_fp = *store.0.vm_store_context().last_wasm_entry_fp.get(); + let stack_chain = (*store.0.vm_store_context().stack_chain.get()).clone(); + + let new_stack_chain = VMStackChain::InitialStack(initial_stack_information); + *store.0.vm_store_context().stack_chain.get() = new_stack_chain; + + let vm_store_context = store.0.vm_store_context(); + + Self { + stack_limit, + last_wasm_exit_pc, + last_wasm_exit_fp, + last_wasm_entry_fp, + stack_chain, + vm_store_context, + } + } + } + + /// This function restores the values stored in this struct. We invoke this + /// function through this type's `Drop` implementation. This ensures that we + /// even restore the values if we unwind the stack (e.g., because we are + /// panicing out of a Wasm execution). + fn exit_wasm(&mut self) { + unsafe { + self.stack_limit.inspect(|limit| { + *(&*self.vm_store_context).stack_limit.get() = *limit; + }); + + *(*self.vm_store_context).last_wasm_exit_fp.get() = self.last_wasm_exit_fp; + *(*self.vm_store_context).last_wasm_exit_pc.get() = self.last_wasm_exit_pc; + *(*self.vm_store_context).last_wasm_entry_fp.get() = self.last_wasm_entry_fp; + *(*self.vm_store_context).stack_chain.get() = self.stack_chain.clone(); + } + } +} + +impl Drop for EntryStoreContext { + fn drop(&mut self) { + self.exit_wasm(); } } diff --git a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs index 14ee5b783856..ed1347e17976 100644 --- a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs +++ b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs @@ -727,6 +727,9 @@ unsafe impl WasmTy for Rooted { | HeapType::I31 | HeapType::Struct | HeapType::ConcreteStruct(_) + | HeapType::Cont + | HeapType::NoCont + | HeapType::ConcreteCont(_) | HeapType::None => bail!( "type mismatch: expected `(ref {ty})`, got `(ref {})`", self._ty(store)?, @@ -821,6 +824,9 @@ unsafe impl WasmTy for ManuallyRooted { | HeapType::I31 | HeapType::Struct | HeapType::ConcreteStruct(_) + | HeapType::Cont + | HeapType::NoCont + | HeapType::ConcreteCont(_) | HeapType::None => bail!( "type mismatch: expected `(ref {ty})`, got `(ref {})`", self._ty(store)?, diff --git a/crates/wasmtime/src/runtime/gc/enabled/structref.rs b/crates/wasmtime/src/runtime/gc/enabled/structref.rs index e111c2f2e633..eb9233b00b15 100644 --- a/crates/wasmtime/src/runtime/gc/enabled/structref.rs +++ b/crates/wasmtime/src/runtime/gc/enabled/structref.rs @@ -594,7 +594,10 @@ unsafe impl WasmTy for Rooted { | HeapType::I31 | HeapType::Array | HeapType::ConcreteArray(_) - | HeapType::None => bail!( + | HeapType::None + | HeapType::NoCont + | HeapType::Cont + | HeapType::ConcreteCont(_) => bail!( "type mismatch: expected `(ref {ty})`, got `(ref {})`", self._ty(store)?, ), @@ -688,7 +691,10 @@ unsafe impl WasmTy for ManuallyRooted { | HeapType::I31 | HeapType::Array | HeapType::ConcreteArray(_) - | HeapType::None => bail!( + | HeapType::None + | HeapType::NoCont + | HeapType::Cont + | HeapType::ConcreteCont(_) => bail!( "type mismatch: expected `(ref {ty})`, got `(ref {})`", self._ty(store)?, ), diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 559abf7ac410..65e9e1692bb2 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -114,6 +114,9 @@ pub use self::async_::CallHookHandler; #[cfg(feature = "async")] use self::async_::*; +use super::vm::stack_switching::stack::VMContinuationStack; +use super::vm::stack_switching::VMContRef; + /// A [`Store`] is a collection of WebAssembly instances and host-defined state. /// /// All WebAssembly instances and items will be attached to and refer to a @@ -306,6 +309,11 @@ pub struct StoreOpaque { engine: Engine, vm_store_context: VMStoreContext, + + // Contains all continuations ever allocated throughout the lifetime of this + // store. + continuations: Vec>, + instances: Vec, #[cfg(feature = "component-model")] num_component_instances: usize, @@ -524,6 +532,7 @@ impl Store { _marker: marker::PhantomPinned, engine: engine.clone(), vm_store_context: Default::default(), + continuations: Vec::new(), instances: Vec::new(), #[cfg(feature = "component-model")] num_component_instances: 0, @@ -1533,6 +1542,7 @@ impl StoreOpaque { assert!(gc_roots_list.is_empty()); self.trace_wasm_stack_roots(gc_roots_list); + self.trace_wasm_continuation_roots(gc_roots_list); self.trace_vmctx_roots(gc_roots_list); self.trace_user_roots(gc_roots_list); @@ -1540,60 +1550,106 @@ impl StoreOpaque { } #[cfg(feature = "gc")] - fn trace_wasm_stack_roots(&mut self, gc_roots_list: &mut GcRootsList) { - use crate::runtime::vm::{Backtrace, SendSyncPtr}; + fn trace_wasm_stack_frame( + &self, + gc_roots_list: &mut GcRootsList, + frame: crate::runtime::vm::Frame, + ) { + use crate::runtime::vm::SendSyncPtr; use core::ptr::NonNull; - log::trace!("Begin trace GC roots :: Wasm stack"); + let pc = frame.pc(); + debug_assert!(pc != 0, "we should always get a valid PC for Wasm frames"); - Backtrace::trace(self, |frame| { - let pc = frame.pc(); - debug_assert!(pc != 0, "we should always get a valid PC for Wasm frames"); + let fp = frame.fp() as *mut usize; + debug_assert!( + !fp.is_null(), + "we should always get a valid frame pointer for Wasm frames" + ); - let fp = frame.fp() as *mut usize; - debug_assert!( - !fp.is_null(), - "we should always get a valid frame pointer for Wasm frames" - ); + let module_info = self + .modules() + .lookup_module_by_pc(pc) + .expect("should have module info for Wasm frame"); - let module_info = self - .modules() - .lookup_module_by_pc(pc) - .expect("should have module info for Wasm frame"); + let stack_map = match module_info.lookup_stack_map(pc) { + Some(sm) => sm, + None => { + log::trace!("No stack map for this Wasm frame"); + return; + } + }; + log::trace!( + "We have a stack map that maps {} bytes in this Wasm frame", + stack_map.frame_size() + ); - let stack_map = match module_info.lookup_stack_map(pc) { - Some(sm) => sm, - None => { - log::trace!("No stack map for this Wasm frame"); - return core::ops::ControlFlow::Continue(()); - } - }; - log::trace!( - "We have a stack map that maps {} bytes in this Wasm frame", - stack_map.frame_size() - ); + let sp = unsafe { stack_map.sp(fp) }; + for stack_slot in unsafe { stack_map.live_gc_refs(sp) } { + let raw: u32 = unsafe { core::ptr::read(stack_slot) }; + log::trace!("Stack slot @ {stack_slot:p} = {raw:#x}"); - let sp = unsafe { stack_map.sp(fp) }; - for stack_slot in unsafe { stack_map.live_gc_refs(sp) } { - let raw: u32 = unsafe { core::ptr::read(stack_slot) }; - log::trace!("Stack slot @ {stack_slot:p} = {raw:#x}"); - - let gc_ref = VMGcRef::from_raw_u32(raw); - if gc_ref.is_some() { - unsafe { - gc_roots_list.add_wasm_stack_root(SendSyncPtr::new( - NonNull::new(stack_slot).unwrap(), - )); - } + let gc_ref = VMGcRef::from_raw_u32(raw); + if gc_ref.is_some() { + unsafe { + gc_roots_list + .add_wasm_stack_root(SendSyncPtr::new(NonNull::new(stack_slot).unwrap())); } } + } + } + + #[cfg(feature = "gc")] + fn trace_wasm_stack_roots(&mut self, gc_roots_list: &mut GcRootsList) { + use crate::runtime::vm::Backtrace; + log::trace!("Begin trace GC roots :: Wasm stack"); + Backtrace::trace(self, |frame| { + self.trace_wasm_stack_frame(gc_roots_list, frame); core::ops::ControlFlow::Continue(()) }); log::trace!("End trace GC roots :: Wasm stack"); } + #[cfg(feature = "gc")] + fn trace_wasm_continuation_roots(&mut self, gc_roots_list: &mut GcRootsList) { + use crate::runtime::vm::Backtrace; + log::trace!("Begin trace GC roots :: continuations"); + + for continuation in &self.continuations { + let state = continuation.common_stack_information.state; + + // FIXME(frank-emrich) In general, it is not enough to just trace + // through the stacks of continuations; we also need to look through + // their `cont.bind` arguments. However, we don't currently have + // enough RTTI information to check if any of the values in the + // buffers used by `cont.bind` are GC values. As a workaround, note + // that we currently disallow cont.bind-ing GC values altogether. + // This way, it is okay not to check them here. + + // Note that we only care about continuations that have state + // `Suspended`. + // - `Running` continuations will be handled by + // `trace_wasm_stack_roots`. + // - For `Parent` continuations, we don't know if they are the + // parent of a running continuation or a suspended one. But it + // does not matter: They will be handled when traversing the stack + // chain starting at either the running one, or the suspended + // continuations below. + // - For `Fresh` continuations, we know that there are no GC values + // on their stack, yet. + if state == crate::vm::stack_switching::VMStackState::Suspended { + Backtrace::trace_suspended_continuation(self, continuation.deref(), |frame| { + self.trace_wasm_stack_frame(gc_roots_list, frame); + core::ops::ControlFlow::Continue(()) + }); + } + } + + log::trace!("End trace GC roots :: continuations"); + } + #[cfg(feature = "gc")] fn trace_vmctx_roots(&mut self, gc_roots_list: &mut GcRootsList) { log::trace!("Begin trace GC roots :: vmctx"); @@ -1885,6 +1941,20 @@ at https://bytecodealliance.org/security. Executor::Native => &crate::runtime::vm::UnwindHost, } } + + /// Allocates a new continuation. Note that we currently don't support + /// deallocating them. Instead, all continuations remain allocated + /// throughout the store's lifetime. + pub fn allocate_continuation(&mut self) -> Result<*mut VMContRef> { + // FIXME(frank-emrich) Do we need to pin this? + let mut continuation = Box::new(VMContRef::empty()); + let stack_size = self.engine.config().stack_switching_config.stack_size; + let stack = VMContinuationStack::new(stack_size)?; + continuation.stack = stack; + let ptr = continuation.deref_mut() as *mut VMContRef; + self.continuations.push(continuation); + Ok(ptr) + } } unsafe impl crate::runtime::vm::VMStore for StoreInner { diff --git a/crates/wasmtime/src/runtime/type_registry.rs b/crates/wasmtime/src/runtime/type_registry.rs index ded58a1f9e99..58fc9c41b538 100644 --- a/crates/wasmtime/src/runtime/type_registry.rs +++ b/crates/wasmtime/src/runtime/type_registry.rs @@ -851,7 +851,7 @@ impl TypeRegistryInner { .struct_layout(s) .into(), ), - wasmtime_environ::WasmCompositeInnerType::Cont(_) => todo!(), // FIXME: #10248 stack switching support. + wasmtime_environ::WasmCompositeInnerType::Cont(_) => None, // FIXME: #10248 stack switching support. }; // Add the type to our slab. diff --git a/crates/wasmtime/src/runtime/types.rs b/crates/wasmtime/src/runtime/types.rs index 2d985acc938c..f1f87a6dadec 100644 --- a/crates/wasmtime/src/runtime/types.rs +++ b/crates/wasmtime/src/runtime/types.rs @@ -154,6 +154,12 @@ impl ValType { /// The `nullref` type, aka `(ref null none)`. pub const NULLREF: Self = ValType::Ref(RefType::NULLREF); + /// The `contref` type, aka `(ref null cont)`. + pub const CONTREF: Self = ValType::Ref(RefType::CONTREF); + + /// The `nullcontref` type, aka. `(ref null nocont)`. + pub const NULLCONTREF: Self = ValType::Ref(RefType::NULLCONTREF); + /// Returns true if `ValType` matches any of the numeric types. (e.g. `I32`, /// `I64`, `F32`, `F64`). #[inline] @@ -236,6 +242,18 @@ impl ValType { ) } + /// Is this the `contref` (aka `(ref null cont)`) type? + #[inline] + pub fn is_contref(&self) -> bool { + matches!( + self, + ValType::Ref(RefType { + is_nullable: true, + heap_type: HeapType::Cont + }) + ) + } + /// Get the underlying reference type, if this value type is a reference /// type. #[inline] @@ -437,6 +455,18 @@ impl RefType { heap_type: HeapType::None, }; + /// The `contref` type, aka `(ref null cont)`. + pub const CONTREF: Self = RefType { + is_nullable: true, + heap_type: HeapType::Cont, + }; + + /// The `nullcontref` type, aka `(ref null nocont)`. + pub const NULLCONTREF: Self = RefType { + is_nullable: true, + heap_type: HeapType::NoCont, + }; + /// Construct a new reference type. pub fn new(is_nullable: bool, heap_type: HeapType) -> RefType { RefType { @@ -697,6 +727,23 @@ pub enum HeapType { /// of `any` and `eq`) and supertypes of the `none` heap type. ConcreteStruct(StructType), + /// A reference to a continuation of a specific, concrete type. + /// + /// These are subtypes of `cont` and supertypes of `nocont`. + ConcreteCont(ContType), + + /// The `cont` heap type represents a reference to any kind of continuation. + /// + /// This is the top type for the continuation objects type hierarchy, and is + /// therefore a supertype of every continuation object. + Cont, + + /// The `nocont` heap type represents the null continuation object. + /// + /// This is the bottom type for the continuation objects type hierarchy, and + /// therefore `nocont` is a subtype of all continuation object types. + NoCont, + /// The abstract `none` heap type represents the null internal reference. /// /// This is the bottom type for the internal type hierarchy, and therefore @@ -720,6 +767,9 @@ impl Display for HeapType { HeapType::ConcreteFunc(ty) => write!(f, "(concrete func {:?})", ty.type_index()), HeapType::ConcreteArray(ty) => write!(f, "(concrete array {:?})", ty.type_index()), HeapType::ConcreteStruct(ty) => write!(f, "(concrete struct {:?})", ty.type_index()), + HeapType::ConcreteCont(ty) => write!(f, "(concrete cont {:?})", ty.type_index()), + HeapType::Cont => write!(f, "cont"), + HeapType::NoCont => write!(f, "nocont"), } } } @@ -745,6 +795,13 @@ impl From for HeapType { } } +impl From for HeapType { + #[inline] + fn from(f: ContType) -> Self { + HeapType::ConcreteCont(f) + } +} + impl HeapType { /// Is this the abstract `extern` heap type? pub fn is_extern(&self) -> bool { @@ -776,6 +833,11 @@ impl HeapType { matches!(self, HeapType::None) } + /// Is this the abstract `cont` heap type? + pub fn is_cont(&self) -> bool { + matches!(self, HeapType::Cont) + } + /// Is this an abstract type? /// /// Types that are not abstract are concrete, user-defined types. @@ -790,7 +852,10 @@ impl HeapType { pub fn is_concrete(&self) -> bool { matches!( self, - HeapType::ConcreteFunc(_) | HeapType::ConcreteArray(_) | HeapType::ConcreteStruct(_) + HeapType::ConcreteFunc(_) + | HeapType::ConcreteArray(_) + | HeapType::ConcreteStruct(_) + | HeapType::ConcreteCont(_) ) } @@ -836,6 +901,21 @@ impl HeapType { self.as_concrete_array().unwrap() } + /// Is this a concrete, user-defined continuation type? + pub fn is_concrete_cont(&self) -> bool { + matches!(self, HeapType::ConcreteCont(_)) + } + + /// Get the underlying concrete, user-defined continuation type, if any. + /// + /// Returns `None` if this is not a concrete continuation type. + pub fn as_concrete_cont(&self) -> Option<&ContType> { + match self { + HeapType::ConcreteCont(f) => Some(f), + _ => None, + } + } + /// Is this a concrete, user-defined struct type? pub fn is_concrete_struct(&self) -> bool { matches!(self, HeapType::ConcreteStruct(_)) @@ -851,6 +931,12 @@ impl HeapType { } } + /// Get the underlying concrete, user-defined type, panicking if this is not + /// a concrete continuation type. + pub fn unwrap_concrete_cont(&self) -> &ContType { + self.as_concrete_cont().unwrap() + } + /// Get the underlying concrete, user-defined type, panicking if this is not /// a concrete struct type. pub fn unwrap_concrete_struct(&self) -> &StructType { @@ -876,6 +962,8 @@ impl HeapType { | HeapType::Struct | HeapType::ConcreteStruct(_) | HeapType::None => HeapType::Any, + + HeapType::Cont | HeapType::ConcreteCont(_) | HeapType::NoCont => HeapType::Cont, } } @@ -883,7 +971,7 @@ impl HeapType { #[inline] pub fn is_top(&self) -> bool { match self { - HeapType::Any | HeapType::Extern | HeapType::Func => true, + HeapType::Any | HeapType::Extern | HeapType::Func | HeapType::Cont => true, _ => false, } } @@ -907,6 +995,8 @@ impl HeapType { | HeapType::Struct | HeapType::ConcreteStruct(_) | HeapType::None => HeapType::None, + + HeapType::Cont | HeapType::ConcreteCont(_) | HeapType::NoCont => HeapType::NoCont, } } @@ -914,7 +1004,7 @@ impl HeapType { #[inline] pub fn is_bottom(&self) -> bool { match self { - HeapType::None | HeapType::NoExtern | HeapType::NoFunc => true, + HeapType::None | HeapType::NoExtern | HeapType::NoFunc | HeapType::NoCont => true, _ => false, } } @@ -952,6 +1042,18 @@ impl HeapType { (HeapType::Func, HeapType::Func) => true, (HeapType::Func, _) => false, + (HeapType::Cont, HeapType::Cont) => true, + (HeapType::Cont, _) => false, + + (HeapType::NoCont, HeapType::NoCont | HeapType::ConcreteCont(_) | HeapType::Cont) => { + true + } + (HeapType::NoCont, _) => false, + + (HeapType::ConcreteCont(_), HeapType::Cont) => true, + (HeapType::ConcreteCont(a), HeapType::ConcreteCont(b)) => a.matches(b), + (HeapType::ConcreteCont(_), _) => false, + ( HeapType::None, HeapType::None @@ -1035,10 +1137,13 @@ impl HeapType { | HeapType::I31 | HeapType::Array | HeapType::Struct + | HeapType::Cont + | HeapType::NoCont | HeapType::None => true, HeapType::ConcreteFunc(ty) => ty.comes_from_same_engine(engine), HeapType::ConcreteArray(ty) => ty.comes_from_same_engine(engine), HeapType::ConcreteStruct(ty) => ty.comes_from_same_engine(engine), + HeapType::ConcreteCont(ty) => ty.comes_from_same_engine(engine), } } @@ -1063,6 +1168,11 @@ impl HeapType { HeapType::ConcreteStruct(a) => { WasmHeapType::ConcreteStruct(EngineOrModuleTypeIndex::Engine(a.type_index())) } + HeapType::Cont => WasmHeapType::Cont, + HeapType::NoCont => WasmHeapType::NoCont, + HeapType::ConcreteCont(c) => { + WasmHeapType::ConcreteCont(EngineOrModuleTypeIndex::Engine(c.type_index())) + } } } @@ -1093,16 +1203,22 @@ impl HeapType { | WasmHeapType::ConcreteArray(EngineOrModuleTypeIndex::Module(_)) | WasmHeapType::ConcreteArray(EngineOrModuleTypeIndex::RecGroup(_)) | WasmHeapType::ConcreteStruct(EngineOrModuleTypeIndex::Module(_)) - | WasmHeapType::ConcreteStruct(EngineOrModuleTypeIndex::RecGroup(_)) => { + | WasmHeapType::ConcreteStruct(EngineOrModuleTypeIndex::RecGroup(_)) + | WasmHeapType::ConcreteCont(EngineOrModuleTypeIndex::Module(_)) + | WasmHeapType::ConcreteCont(EngineOrModuleTypeIndex::RecGroup(_)) => { panic!("HeapType::from_wasm_type on non-canonicalized-for-runtime-usage heap type") } - - WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapType::Cont => HeapType::Cont, + WasmHeapType::NoCont => HeapType::NoCont, + WasmHeapType::ConcreteCont(EngineOrModuleTypeIndex::Engine(idx)) => { + HeapType::ConcreteCont(ContType::from_shared_type_index(engine, *idx)) + } } } pub(crate) fn as_registered_type(&self) -> Option<&RegisteredType> { match self { + HeapType::ConcreteCont(c) => Some(&c.registered_type), HeapType::ConcreteFunc(f) => Some(&f.registered_type), HeapType::ConcreteArray(a) => Some(&a.registered_type), HeapType::ConcreteStruct(a) => Some(&a.registered_type), @@ -1116,6 +1232,8 @@ impl HeapType { | HeapType::I31 | HeapType::Array | HeapType::Struct + | HeapType::Cont + | HeapType::NoCont | HeapType::None => None, } } @@ -1125,6 +1243,7 @@ impl HeapType { match self.top() { Self::Any | Self::Extern => true, Self::Func => false, + Self::Cont => false, ty => unreachable!("not a top type: {ty:?}"), } } @@ -2413,6 +2532,56 @@ impl FuncType { } } +// Continuation types +/// A WebAssembly continuation descriptor. +#[derive(Debug, Clone, Hash)] +pub struct ContType { + registered_type: RegisteredType, +} + +impl ContType { + /// Get the engine that this function type is associated with. + pub fn engine(&self) -> &Engine { + self.registered_type.engine() + } + + pub(crate) fn comes_from_same_engine(&self, engine: &Engine) -> bool { + Engine::same(self.registered_type.engine(), engine) + } + + pub(crate) fn type_index(&self) -> VMSharedTypeIndex { + self.registered_type.index() + } + + /// Does this continuation type match the other continuation type? + /// + /// That is, is this continuation type a subtype of the other continuation type? + /// + /// # Panics + /// + /// Panics if either type is associated with a different engine from the + /// other. + pub fn matches(&self, other: &ContType) -> bool { + assert!(self.comes_from_same_engine(other.engine())); + + // Avoid matching on structure for subtyping checks when we have + // precisely the same type. + // TODO(dhil): Implement subtype check later. + self.type_index() == other.type_index() + } + + pub(crate) fn from_shared_type_index(engine: &Engine, index: VMSharedTypeIndex) -> ContType { + let ty = RegisteredType::root(engine, index).expect( + "VMSharedTypeIndex is not registered in the Engine! Wrong \ + engine? Didn't root the index somewhere?", + ); + assert!(ty.is_cont()); + Self { + registered_type: ty, + } + } +} + // Global Types /// A WebAssembly global descriptor. diff --git a/crates/wasmtime/src/runtime/values.rs b/crates/wasmtime/src/runtime/values.rs index 3d606d795900..0207d35f9094 100644 --- a/crates/wasmtime/src/runtime/values.rs +++ b/crates/wasmtime/src/runtime/values.rs @@ -280,6 +280,11 @@ impl Val { HeapType::NoFunc => Ref::Func(None), + HeapType::NoCont | HeapType::ConcreteCont(_) | HeapType::Cont => { + // TODO(#10248): Required to support stack switching in the embedder API. + unimplemented!() + } + HeapType::Extern => ExternRef::_from_raw(store, raw.get_externref()).into(), HeapType::NoExtern => Ref::Extern(None), diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index 189f93dc86a5..eb751fb45675 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -42,6 +42,7 @@ mod vmcontext; #[cfg(feature = "threads")] mod parking_spot; +pub mod stack_switching; // Note that `debug_builtins` here is disabled with a feature or a lack of a // native compilation backend because it's only here to assist in debugging @@ -95,6 +96,7 @@ pub use crate::runtime::vm::vmcontext::{ VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, VMStoreContext, VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw, }; + pub use send_sync_ptr::SendSyncPtr; mod module_id; diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs index 1c26e59b15ac..12c78116e79c 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs @@ -8,7 +8,6 @@ use crate::runtime::vm::{ SendSyncPtr, Table, }; use crate::{prelude::*, vm::HostAlignedByteCount}; -use std::mem; use std::ptr::NonNull; use wasmtime_environ::{Module, Tunables}; @@ -31,7 +30,7 @@ impl TablePool { /// Create a new `TablePool`. pub fn new(config: &PoolingInstanceAllocatorConfig) -> Result { let table_size = HostAlignedByteCount::new_rounded_up( - mem::size_of::<*mut u8>() + crate::runtime::vm::table::MAX_TABLE_ELEM_SIZE .checked_mul(config.limits.table_elements) .ok_or_else(|| anyhow!("table size exceeds addressable memory"))?, )?; @@ -133,13 +132,15 @@ impl TablePool { match (|| { let base = self.get(allocation_index); + let element_size = crate::vm::table::wasm_to_table_type(ty.ref_type).element_size(); + unsafe { - commit_pages(base, self.table_elements * mem::size_of::<*mut u8>())?; + commit_pages(base, self.table_elements * element_size)?; } let ptr = NonNull::new(std::ptr::slice_from_raw_parts_mut( base.cast(), - self.table_elements * mem::size_of::<*mut u8>(), + self.table_elements * element_size, )) .unwrap(); unsafe { @@ -198,8 +199,10 @@ impl TablePool { // doesn't overflow? The only check that exists is for the boundary // condition that table.size() * mem::size_of::<*mut u8>() is less than // a host page smaller than usize::MAX. - let size = HostAlignedByteCount::new_rounded_up(table.size() * mem::size_of::<*mut u8>()) - .expect("table entry size doesn't overflow"); + let size = HostAlignedByteCount::new_rounded_up( + table.size() * table.element_type().element_size(), + ) + .expect("table entry size doesn't overflow"); // `memset` the first `keep_resident` bytes. let size_to_memset = size.min(self.keep_resident); diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index 0447c4c29b58..e489465579b2 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -54,6 +54,8 @@ //! } //! ``` +use super::stack_switching::VMContObj; +use super::stack_switching::VMContRef; use crate::prelude::*; use crate::runtime::vm::table::{Table, TableElementType}; use crate::runtime::vm::vmcontext::VMFuncRef; @@ -219,6 +221,7 @@ unsafe fn table_grow_func_ref( let element = match instance.table_element_type(table_index) { TableElementType::Func => NonNull::new(init_value.cast::()).into(), TableElementType::GcRef => unreachable!(), + TableElementType::Cont => unreachable!(), }; let result = instance @@ -248,6 +251,40 @@ unsafe fn table_grow_gc_ref( .clone_gc_ref(&r) }) .into(), + TableElementType::Cont => unreachable!(), + }; + + let result = instance + .table_grow(store, table_index, delta, element)? + .map(AllocationSize); + Ok(result) +} + +unsafe fn table_grow_cont_obj( + store: &mut dyn VMStore, + instance: &mut Instance, + table_index: u32, + delta: u64, + // The following two values together form the intitial Option. + // A None value is indicated by the pointer being null. + init_value_contref: *mut u8, + init_value_revision: u64, +) -> Result> { + use core::ptr::NonNull; + let init_value = if init_value_contref.is_null() { + None + } else { + // SAFETY: We just checked that the pointer is non-null + let contref = NonNull::new_unchecked(init_value_contref as *mut VMContRef); + let contobj = VMContObj::new(contref, init_value_revision); + Some(contobj) + }; + + let table_index = TableIndex::from_u32(table_index); + + let element = match instance.table_element_type(table_index) { + TableElementType::Cont => init_value.into(), + _ => panic!("Wrong table growing function"), }; let result = instance @@ -274,6 +311,7 @@ unsafe fn table_fill_func_ref( Ok(()) } TableElementType::GcRef => unreachable!(), + TableElementType::Cont => unreachable!(), } } @@ -297,6 +335,38 @@ unsafe fn table_fill_gc_ref( table.fill(Some(gc_store), dst, gc_ref.into(), len)?; Ok(()) } + + TableElementType::Cont => unreachable!(), + } +} + +unsafe fn table_fill_cont_obj( + store: &mut dyn VMStore, + instance: &mut Instance, + table_index: u32, + dst: u64, + value_contref: *mut u8, + value_revision: u64, + len: u64, +) -> Result<()> { + use core::ptr::NonNull; + let table_index = TableIndex::from_u32(table_index); + let table = &mut *instance.get_table(table_index); + match table.element_type() { + TableElementType::Cont => { + let contobj = if value_contref.is_null() { + None + } else { + // SAFETY: We just checked that the pointer is non-null + let contref = NonNull::new_unchecked(value_contref as *mut VMContRef); + let contobj = VMContObj::new(contref, value_revision); + Some(contobj) + }; + + table.fill(store.optional_gc_store_mut()?, dst, contobj.into(), len)?; + Ok(()) + } + _ => panic!("Wrong table filling function"), } } @@ -1359,3 +1429,36 @@ pub mod relocs { } } } + +// Builtins for continuations. These are thin wrappers around the +// respective definitions in stack_switching.rs. +fn cont_new( + store: &mut dyn VMStore, + instance: &mut Instance, + func: *mut u8, + param_count: u32, + result_count: u32, +) -> Result, TrapReason> { + let ans = + crate::vm::stack_switching::cont_new(store, instance, func, param_count, result_count)?; + Ok(Some(AllocationSize(ans.cast::() as usize))) +} + +fn delete_me_print_str(_store: &mut dyn VMStore, _instance: &mut Instance, s: *const u8, len: u64) { + let len = + usize::try_from(len).map_err(|_error| TrapReason::User(anyhow::anyhow!("len too large!"))); + let str = unsafe { core::slice::from_raw_parts(s, len.unwrap()) }; + let _s = core::str::from_utf8(str).unwrap(); + #[cfg(feature = "std")] + print!("{_s}") +} + +fn delete_me_print_int(_store: &mut dyn VMStore, _instance: &mut Instance, _arg: u64) { + #[cfg(feature = "std")] + print!("{_arg}") +} + +fn delete_me_print_pointer(_store: &mut dyn VMStore, _instance: &mut Instance, _arg: *const u8) { + #[cfg(feature = "std")] + print!("{_arg:p}") +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs new file mode 100644 index 000000000000..59203e817f4a --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -0,0 +1,705 @@ +//! This module contains the runtime components of the implementation of the +//! stack switching proposal. + +use core::{marker::PhantomPinned, ptr::NonNull}; + +use stack::VMContinuationStack; +#[allow(unused)] +use wasmtime_environ::{debug_println, stack_switching::ENABLE_DEBUG_PRINTING}; + +use crate::vm::{Instance, TrapReason, VMFuncRef, VMStore}; +use crate::ValRaw; + +pub mod stack; + +/// A continuation object is a handle to a continuation reference +/// (i.e. an actual stack). A continuation object only be consumed +/// once. The linearity is checked dynamically in the generated code +/// by comparing the revision witness embedded in the pointer to the +/// actual revision counter on the continuation reference. +/// +/// In the optimized implementation, the continuation logically +/// represented by a VMContObj not only encompasses the pointed-to +/// VMContRef, but also all of its parents: +/// +/// ```text +/// +/// +----------------+ +/// +-->| VMContRef | +/// | +----------------+ +/// | ^ +/// | | parent +/// | | +/// | +----------------+ +/// | | VMContRef | +/// | +----------------+ +/// | ^ +/// | | parent +/// last ancestor | | +/// | +----------------+ +/// +---| VMContRef | <-- VMContObj +/// +----------------+ +/// ``` +/// +/// For performance reasons, the VMContRef at the bottom of this chain +/// (i.e., the one pointed to by the VMContObj) has a pointer to the +/// other end of the chain (i.e., its last ancestor). +// FIXME(frank-emrich) Does this actually need to be 16-byte aligned any +// more? Now that we use I128 on the Cranelift side (see +// [wasmtime_cranelift::stack_switching::fatpointer::pointer_type]), it +// should be fine to use the natural alignment of the type. +#[repr(C, align(16))] +#[derive(Debug, Clone, Copy)] +pub struct VMContObj { + pub revision: u64, + pub contref: NonNull, +} + +impl VMContObj { + pub fn new(contref: NonNull, revision: u64) -> Self { + Self { contref, revision } + } +} + +unsafe impl Send for VMContObj {} +unsafe impl Sync for VMContObj {} + +#[test] +fn null_pointer_optimization() { + // The Rust spec does not technically guarantee that the null pointer + // optimization applies to a struct containing a `NonNull`. + assert_eq!( + core::mem::size_of::>(), + core::mem::size_of::() + ); +} + +/// This type is used to save (and subsequently restore) a subset of the data in +/// `VMRuntimeLimits`. See documentation of `VMStackChain` for the exact uses. +#[repr(C)] +#[derive(Debug, Default, Clone)] +pub struct VMStackLimits { + /// Saved version of `stack_limit` field of `VMRuntimeLimits` + pub stack_limit: usize, + /// Saved version of `last_wasm_entry_fp` field of `VMRuntimeLimits` + pub last_wasm_entry_fp: usize, +} + +#[test] +fn check_vm_stack_limits_offsets() { + use core::mem::offset_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + offset_of!(VMStackLimits, stack_limit), + usize::from(offsets.ptr.vmstack_limits_stack_limit()) + ); + assert_eq!( + offset_of!(VMStackLimits, last_wasm_entry_fp), + usize::from(offsets.ptr.vmstack_limits_last_wasm_entry_fp()) + ); +} + +/// This type represents "common" information that we need to save both for the +/// initial stack and each continuation. +#[repr(C)] +#[derive(Debug, Clone)] +pub struct VMCommonStackInformation { + /// Saves subset of `VMRuntimeLimits` for this stack. See documentation of + /// `VMStackChain` for the exact uses. + pub limits: VMStackLimits, + /// For the initial stack, this field must only have one of the following values: + /// - Running + /// - Parent + pub state: VMStackState, + + /// Only in use when state is `Parent`. Otherwise, the list must be empty. + /// + /// Represents the handlers that this stack installed when resume-ing a + /// continuation. + /// + /// Note that for any resume instruction, we can re-order the handler + /// clauses without changing behavior such that all the suspend handlers + /// come first, followed by all the switch handler (while maintaining the + /// original ordering within the two groups). + /// Thus, we assume that the given resume instruction has the following + /// shape: + /// + /// (resume $ct + /// (on $tag_0 $block_0) ... (on $tag_{n-1} $block_{n-1}) + /// (on $tag_n switch) ... (on $tag_m switch) + /// ) + /// + /// On resume, the handler list is then filled with m + 1 (i.e., one per + /// handler clause) entries such that the i-th entry, using 0-based + /// indexing, is the identifier of $tag_i (represented as *mut + /// VMTagDefinition). + /// Further, `first_switch_handler_index` (see below) is set to n (i.e., the + /// 0-based index of the first switch handler). + /// + /// Note that the actual data buffer (i.e., the one `handler.data` points + /// to) is always allocated on the stack that this `CommonStackInformation` + /// struct describes. + pub handlers: VMHandlerList, + + /// Only used when state is `Parent`. See documentation of `handlers` above. + pub first_switch_handler_index: u32, +} + +impl VMCommonStackInformation { + /// Default value with state set to `Running` + pub fn running_default() -> Self { + Self { + limits: VMStackLimits::default(), + state: VMStackState::Running, + handlers: VMHandlerList::empty(), + first_switch_handler_index: 0, + } + } +} + +#[test] +fn check_vm_common_stack_information_offsets() { + use core::mem::offset_of; + use std::mem::size_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + size_of::(), + usize::from(offsets.ptr.size_of_vmcommon_stack_information()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, limits), + usize::from(offsets.ptr.vmcommon_stack_information_limits()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, state), + usize::from(offsets.ptr.vmcommon_stack_information_state()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, handlers), + usize::from(offsets.ptr.vmcommon_stack_information_handlers()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, first_switch_handler_index), + usize::from( + offsets + .ptr + .vmcommon_stack_information_first_switch_handler_index() + ) + ); +} + +impl VMStackLimits { + /// Default value, but uses the given value for `stack_limit`. + pub fn with_stack_limit(stack_limit: usize) -> Self { + Self { + stack_limit, + ..Default::default() + } + } +} + +#[repr(C)] +#[derive(Debug, Clone)] +/// Reference to a stack-allocated buffer ("array"), storing data of some type +/// `T`. +pub struct VMArray { + /// Number of currently occupied slots. + pub length: u32, + /// Number of slots in the data buffer. Note that this is *not* the size of + /// the buffer in bytes! + pub capacity: u32, + /// The actual data buffer + pub data: *mut T, +} + +impl VMArray { + /// Creates empty `Array` + pub fn empty() -> Self { + Self { + length: 0, + capacity: 0, + data: core::ptr::null_mut(), + } + } + + /// Makes `Array` empty. + pub fn clear(&mut self) { + *self = Self::empty(); + } +} + +#[test] +fn check_vm_array_offsets() { + use core::mem::offset_of; + use std::mem::size_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + // Note that the type parameter has no influence on the size and offsets. + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + size_of::>(), + usize::from(offsets.ptr.size_of_vmarray()) + ); + assert_eq!( + offset_of!(VMArray<()>, length), + usize::from(offsets.ptr.vmarray_length()) + ); + assert_eq!( + offset_of!(VMArray<()>, capacity), + usize::from(offsets.ptr.vmarray_capacity()) + ); + assert_eq!( + offset_of!(VMArray<()>, data), + usize::from(offsets.ptr.vmarray_data()) + ); +} + +/// Type used for passing payloads to and from continuations. The actual type +/// argument should be wasmtime::runtime::vm::vmcontext::ValRaw, but we don't +/// have access to that here. +pub type VMPayloads = VMArray; + +/// Type for a list of handlers, represented by the handled tag. Thus, the +/// stored data is actually `*mut VMTagDefinition`, but we don't havr access to +/// that here. +pub type VMHandlerList = VMArray<*mut u8>; + +/// The main type representing a continuation. +#[repr(C)] +pub struct VMContRef { + /// The `CommonStackInformation` of this continuation's stack. + pub common_stack_information: VMCommonStackInformation, + + /// The parent of this continuation, which may be another continuation, the + /// initial stack, or absent (in case of a suspended continuation). + pub parent_chain: VMStackChain, + + /// Only used if `common_stack_information.state` is `Suspended` or `Fresh`. In + /// that case, this points to the end of the stack chain (i.e., the + /// continuation in the parent chain whose own `parent_chain` field is + /// `VMStackChain::Absent`). + /// Note that this may be a pointer to iself (if the state is `Fresh`, this is always the case). + pub last_ancestor: *mut VMContRef, + + /// Revision counter. + pub revision: u64, + + /// The underlying stack. + pub stack: VMContinuationStack, + + /// Used to store only + /// 1. The arguments to the function passed to cont.new + /// 2. The return values of that function + /// + /// Note that the actual data buffer (i.e., the one `args.data` points + /// to) is always allocated on this continuation's stack. + pub args: VMPayloads, + + /// Once a continuation has been suspended (using suspend or switch), + /// this buffer is used to pass payloads to and from the continuation. + /// More concretely, it is used to + /// - Pass payloads from a suspend instruction to the corresponding handler. + /// - Pass payloads to a continuation using cont.bind or resume + /// - Pass payloads to the continuation being switched to when using switch. + /// + /// Note that the actual data buffer (i.e., the one `values.data` points + /// to) is always allocated on this continuation's stack. + pub values: VMPayloads, + + /// Tell the compiler that this structure has potential self-references + /// through the `last_ancestor` pointer. + _marker: core::marker::PhantomPinned, +} + +impl VMContRef { + pub fn fiber_stack(&self) -> &VMContinuationStack { + &self.stack + } + + pub fn detach_stack(&mut self) -> VMContinuationStack { + core::mem::replace(&mut self.stack, VMContinuationStack::unallocated()) + } + + /// This is effectively a `Default` implementation, without calling it + /// so. Used to create `VMContRef`s when initializing pooling allocator. + #[allow(clippy::cast_possible_truncation)] + pub fn empty() -> Self { + let limits = VMStackLimits::with_stack_limit(Default::default()); + let state = VMStackState::Fresh; + let handlers = VMHandlerList::empty(); + let common_stack_information = VMCommonStackInformation { + limits, + state, + handlers, + first_switch_handler_index: 0, + }; + let parent_chain = VMStackChain::Absent; + let last_ancestor = core::ptr::null_mut(); + let stack = VMContinuationStack::unallocated(); + let args = VMPayloads::empty(); + let values = VMPayloads::empty(); + let revision = 0; + let _marker = PhantomPinned; + + Self { + common_stack_information, + parent_chain, + last_ancestor, + stack, + args, + values, + revision, + _marker, + } + } +} + +impl Drop for VMContRef { + fn drop(&mut self) { + // Note that continuation references do not own their parents, and we + // don't drop them here. + + // We would like to enforce the invariant that any continuation that + // was created for a cont.new (rather than, say, just living in a + // pool and never being touched), either ran to completion or was + // cancelled. But failing to do so should yield a custom error, + // instead of panicking here. + } +} + +// These are required so the WasmFX pooling allocator can store a Vec of +// `VMContRef`s. +unsafe impl Send for VMContRef {} +unsafe impl Sync for VMContRef {} + +#[test] +fn check_vm_contref_offsets() { + use core::mem::offset_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + offset_of!(VMContRef, common_stack_information), + usize::from(offsets.ptr.vmcontref_common_stack_information()) + ); + assert_eq!( + offset_of!(VMContRef, parent_chain), + usize::from(offsets.ptr.vmcontref_parent_chain()) + ); + assert_eq!( + offset_of!(VMContRef, last_ancestor), + usize::from(offsets.ptr.vmcontref_last_ancestor()) + ); + // Some 32-bit platforms need this to be 8-byte aligned, some don't. + // So we need to make sure it always is, without padding. + assert_eq!(u8::vmcontref_revision(&4) % 8, 0); + assert_eq!(u8::vmcontref_revision(&8) % 8, 0); + assert_eq!( + offset_of!(VMContRef, revision), + usize::from(offsets.ptr.vmcontref_revision()) + ); + assert_eq!( + offset_of!(VMContRef, stack), + usize::from(offsets.ptr.vmcontref_stack()) + ); + assert_eq!( + offset_of!(VMContRef, args), + usize::from(offsets.ptr.vmcontref_args()) + ); + assert_eq!( + offset_of!(VMContRef, values), + usize::from(offsets.ptr.vmcontref_values()) + ); +} + +/// Implements `cont.new` instructions (i.e., creation of continuations). +#[inline(always)] +pub fn cont_new( + store: &mut dyn VMStore, + instance: &mut Instance, + func: *mut u8, + param_count: u32, + result_count: u32, +) -> Result<*mut VMContRef, TrapReason> { + let caller_vmctx = instance.vmctx(); + + let stack_size = store.engine().config().stack_switching_config.stack_size; + + let contref = store.allocate_continuation()?; + let contref = unsafe { contref.as_mut().unwrap() }; + + let tsp = contref.stack.top().unwrap(); + contref.parent_chain = VMStackChain::Absent; + // The continuation is fresh, which is a special case of being suspended. + // Thus we need to set the correct end of the continuation chain: itself. + contref.last_ancestor = contref; + + // The initialization function will allocate the actual args/return value buffer and + // update this object (if needed). + let contref_args_ptr = &mut contref.args as *mut _ as *mut VMArray; + + contref.stack.initialize( + func.cast::(), + caller_vmctx.as_ptr(), + contref_args_ptr, + param_count, + result_count, + ); + + // Now that the initial stack pointer was set by the initialization + // function, use it to determine stack limit. + let stack_pointer = contref.stack.control_context_stack_pointer(); + // Same caveat regarding stack_limit here as descibed in + // `wasmtime::runtime::func::EntryStoreContext::enter_wasm`. + let wasm_stack_limit = core::cmp::max( + stack_pointer - store.engine().config().max_wasm_stack, + tsp as usize - stack_size, + ); + let limits = VMStackLimits::with_stack_limit(wasm_stack_limit); + let csi = &mut contref.common_stack_information; + csi.state = VMStackState::Fresh; + csi.limits = limits; + + debug_println!("Created contref @ {:p}", contref); + Ok(contref) +} + +/// This type represents a linked lists ("chain") of stacks, where the a +/// node's successor denotes its parent. +/// A additionally, a `CommonStackInformation` object is associated with +/// each stack in the list. +/// Here, a "stack" is one of the following: +/// - A continuation (i.e., created with cont.new). +/// - The initial stack. This is the stack that we were on when entering +/// Wasm (i.e., when executing +/// `crate::runtime::func::invoke_wasm_and_catch_traps`). +/// This stack never has a parent. +/// In terms of the memory allocation that this stack resides on, it will +/// usually be the main stack, but doesn't have to: If we are running +/// inside a continuation while executing a host call, which in turn +/// re-renters Wasm, the initial stack is actually the stack of that +/// continuation. +/// +/// Note that the linked list character of `VMStackChain` arises from the fact +/// that `VMStackChain::Continuation` variants have a pointer to a +/// `VMContRef`, which in turn has a `parent_chain` value of type +/// `VMStackChain`. This is how the stack chain reflects the parent-child +/// relationships between continuations/stacks. This also shows how the +/// initial stack (mentioned above) cannot have a parent. +/// +/// There are generally two uses of `VMStackChain`: +/// +/// 1. The `stack_chain` field in the `StoreOpaque` contains such a +/// chain of stacks, where the head of the list denotes the stack that is +/// currently executing (either a continuation or the initial stack). Note +/// that in this case, the linked list must contain 0 or more `Continuation` +/// elements, followed by a final `InitialStack` element. In particular, +/// this list always ends with `InitialStack` and never contains an `Absent` +/// variant. +/// +/// 2. When a continuation is suspended, its chain of parents eventually +/// ends with an `Absent` variant in its `parent_chain` field. Note that a +/// suspended continuation never appears in the stack chain in the +/// VMContext! +/// +/// +/// As mentioned before, each stack in a `VMStackChain` has a corresponding +/// `CommonStackInformation` object. For continuations, this is stored in +/// the `common_stack_information` field of the corresponding `VMContRef`. +/// For the initial stack, the `InitialStack` variant contains a pointer to +/// a `CommonStackInformation`. The latter will be allocated allocated on +/// the stack frame that executed by `invoke_wasm_and_catch_traps`. +/// +/// The following invariants hold for these `VMStackLimits` objects, +/// and the data in `VMRuntimeLimits`. +/// +/// Currently executing stack: For the currently executing stack (i.e., the +/// stack that is at the head of the store's `stack_chain` list), the +/// associated `VMStackLimits` object contains stale/undefined data. Instead, +/// the live data describing the limits for the currently executing stack is +/// always maintained in `VMRuntimeLimits`. Note that as a general rule +/// independently from any execution of continuations, the `last_wasm_exit*` +/// fields in the `VMRuntimeLimits` contain undefined values while executing +/// wasm. +/// +/// Parents of currently executing stack: For stacks that appear in the tail +/// of the store's `stack_chain` list (i.e., stacks that are not currently +/// executing themselves, but are an ancestor of the currently executing +/// stack), we have the following: All the fields in the stack's +/// `VMStackLimits` are valid, describing the stack's stack limit, and +/// pointers where executing for that stack entered and exited WASM. +/// +/// Suspended continuations: For suspended continuations (including their +/// ancestors), we have the following. Note that the initial stack can never +/// be in this state. The `stack_limit` and `last_enter_wasm_sp` fields of +/// the corresponding `VMStackLimits` object contain valid data, while the +/// `last_exit_wasm_*` fields contain arbitrary values. There is only one +/// exception to this: Note that a continuation that has been created with +/// cont.new, but never been resumed so far, is considered "suspended". +/// However, its `last_enter_wasm_sp` field contains undefined data. This is +/// justified, because when resume-ing a continuation for the first time, a +/// native-to-wasm trampoline is called, which sets up the +/// `last_wasm_entry_sp` in the `VMRuntimeLimits` with the correct value, +/// thus restoring the necessary invariant. +#[derive(Debug, Clone, PartialEq)] +#[repr(usize, C)] +pub enum VMStackChain { + /// For suspended continuations, denotes the end of their chain of + /// ancestors. + Absent = wasmtime_environ::stack_switching::STACK_CHAIN_ABSENT_DISCRIMINANT, + /// Represents the initial stack (i.e., where we entered Wasm from the + /// host by executing + /// `crate::runtime::func::invoke_wasm_and_catch_traps`). Therefore, it + /// does not have a parent. The `CommonStackInformation` that this + /// variant points to is stored in the stack frame of + /// `invoke_wasm_and_catch_traps`. + InitialStack(*mut VMCommonStackInformation) = + wasmtime_environ::stack_switching::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT, + /// Represents a continuation's stack. + Continuation(*mut VMContRef) = + wasmtime_environ::stack_switching::STACK_CHAIN_CONTINUATION_DISCRIMINANT, +} + +impl VMStackChain { + /// Indicates if `self` is a `InitialStack` variant. + pub fn is_initial_stack(&self) -> bool { + matches!(self, VMStackChain::InitialStack(_)) + } + + /// Returns an iterator over the continuations in this chain. + /// We don't implement `IntoIterator` because our iterator is unsafe, so at + /// least this gives us some way of indicating this, even though the actual + /// unsafety lies in the `next` function. + /// + /// # Safety + /// + /// This function is not unsafe per see, but it returns an object + /// whose usage is unsafe. + pub unsafe fn into_continuation_iter(self) -> ContinuationIterator { + ContinuationIterator(self) + } + + /// Returns an iterator over the stack limits in this chain. + /// We don't implement `IntoIterator` because our iterator is unsafe, so at + /// least this gives us some way of indicating this, even though the actual + /// unsafety lies in the `next` function. + /// + /// # Safety + /// + /// This function is not unsafe per see, but it returns an object + /// whose usage is unsafe. + pub unsafe fn into_stack_limits_iter(self) -> StackLimitsIterator { + StackLimitsIterator(self) + } +} + +#[test] +fn check_vm_stack_chain_offsets() { + use std::mem::size_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + size_of::(), + usize::from(offsets.ptr.size_of_vmstack_chain()) + ); +} + +/// Iterator for Continuations in a stack chain. +pub struct ContinuationIterator(VMStackChain); + +/// Iterator for VMStackLimits in a stack chain. +pub struct StackLimitsIterator(VMStackChain); + +impl Iterator for ContinuationIterator { + type Item = *mut VMContRef; + + fn next(&mut self) -> Option { + match self.0 { + VMStackChain::Absent | VMStackChain::InitialStack(_) => None, + VMStackChain::Continuation(ptr) => { + let continuation = unsafe { ptr.as_mut().unwrap() }; + self.0 = continuation.parent_chain.clone(); + Some(ptr) + } + } + } +} + +impl Iterator for StackLimitsIterator { + type Item = *mut VMStackLimits; + + fn next(&mut self) -> Option { + match self.0 { + VMStackChain::Absent => None, + VMStackChain::InitialStack(csi) => { + let stack_limits = unsafe { &mut (*csi).limits } as *mut VMStackLimits; + self.0 = VMStackChain::Absent; + Some(stack_limits) + } + VMStackChain::Continuation(ptr) => { + let continuation = unsafe { ptr.as_mut().unwrap() }; + let stack_limits = + (&mut continuation.common_stack_information.limits) as *mut VMStackLimits; + self.0 = continuation.parent_chain.clone(); + Some(stack_limits) + } + } + } +} + +/// Encodes the life cycle of a `VMContRef`. +#[derive(Debug, Clone, Copy, PartialEq)] +#[repr(u32)] +pub enum VMStackState { + /// The `VMContRef` has been created, but neither `resume` or `switch` has ever been + /// called on it. During this stage, we may add arguments using `cont.bind`. + Fresh = wasmtime_environ::stack_switching::STACK_STATE_FRESH_DISCRIMINANT, + /// The continuation is running, meaning that it is the one currently + /// executing code. + Running = wasmtime_environ::stack_switching::STACK_STATE_RUNNING_DISCRIMINANT, + /// The continuation is suspended because it executed a resume instruction + /// that has not finished yet. In other words, it became the parent of + /// another continuation (which may itself be `Running`, a `Parent`, or + /// `Suspended`). + Parent = wasmtime_environ::stack_switching::STACK_STATE_PARENT_DISCRIMINANT, + /// The continuation was suspended by a `suspend` or `switch` instruction. + Suspended = wasmtime_environ::stack_switching::STACK_STATE_SUSPENDED_DISCRIMINANT, + /// The function originally passed to `cont.new` has returned normally. + /// Note that there is no guarantee that a VMContRef will ever + /// reach this status, as it may stay suspended until being dropped. + Returned = wasmtime_environ::stack_switching::STACK_STATE_RETURNED_DISCRIMINANT, +} + +/// Universal control effect. This structure encodes return signal, resume +/// signal, suspension signal, and the handler to suspend to in a single variant +/// type. This instance is used at runtime. There is a codegen counterpart in +/// `cranelift/src/stack-switching/control_effect.rs`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u32)] +#[allow(dead_code)] +pub enum ControlEffect { + /// Used to signal that a continuation has returned and control switches + /// back to the parent. + Return = wasmtime_environ::stack_switching::CONTROL_EFFECT_RETURN_DISCRIMINANT, + /// Used to signal to a continuation that it is being resumed. + Resume = wasmtime_environ::stack_switching::CONTROL_EFFECT_RESUME_DISCRIMINANT, + /// Used to signal that a continuation has invoked a `suspend` instruction. + Suspend { + /// The index of the handler to be used in the parent continuation to + /// switch back to. + handler_index: u32, + } = wasmtime_environ::stack_switching::CONTROL_EFFECT_SUSPEND_DISCRIMINANT, + /// Used to signal that a continuation has invoked a `suspend` instruction. + Switch = wasmtime_environ::stack_switching::CONTROL_EFFECT_SWITCH_DISCRIMINANT, +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs new file mode 100644 index 000000000000..12637585df42 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs @@ -0,0 +1,119 @@ +//! This module contains a modified version of the `wasmtime_fiber` crate, +//! specialized for executing stack switching continuations. + +#![allow(missing_docs)] + +use anyhow::Result; +use core::ops::Range; + +use crate::runtime::vm::stack_switching::VMArray; +use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; + +cfg_if::cfg_if! { + if #[cfg(all(feature = "stack-switching",unix, target_arch = "x86_64"))] { + pub mod unix; + use unix as imp; + } else { + pub mod dummy; + use dummy as imp; + } +} + +/// Represents an execution stack to use for a fiber. +#[derive(Debug)] +#[repr(C)] +pub struct VMContinuationStack(imp::VMContinuationStack); + +impl VMContinuationStack { + /// Creates a new fiber stack of the given size. + pub fn new(size: usize) -> Result { + Ok(Self(imp::VMContinuationStack::new(size)?)) + } + + /// Returns a stack of size 0. + pub fn unallocated() -> Self { + Self(imp::VMContinuationStack::unallocated()) + } + + /// Is this stack unallocated/of size 0? + pub fn is_unallocated(&self) -> bool { + imp::VMContinuationStack::is_unallocated(&self.0) + } + + /// Creates a new fiber stack with the given pointer to the bottom of the + /// stack plus the byte length of the stack. + /// + /// The `bottom` pointer should be addressable for `len` bytes. The page + /// beneath `bottom` should be unmapped as a guard page. + /// + /// # Safety + /// + /// This is unsafe because there is no validation of the given pointer. + /// + /// The caller must properly allocate the stack space with a guard page and + /// make the pages accessible for correct behavior. + pub unsafe fn from_raw_parts(bottom: *mut u8, guard_size: usize, len: usize) -> Result { + Ok(Self(imp::VMContinuationStack::from_raw_parts( + bottom, guard_size, len, + )?)) + } + + /// Is this a manually-managed stack created from raw parts? If so, it is up + /// to whoever created it to manage the stack's memory allocation. + pub fn is_from_raw_parts(&self) -> bool { + self.0.is_from_raw_parts() + } + + /// Gets the top of the stack. + /// + /// Returns `None` if the platform does not support getting the top of the + /// stack. + pub fn top(&self) -> Option<*mut u8> { + self.0.top() + } + + /// Returns the range of where this stack resides in memory if the platform + /// supports it. + pub fn range(&self) -> Option> { + self.0.range() + } + + /// Returns the instruction pointer stored in the Fiber's ControlContext. + pub fn control_context_instruction_pointer(&self) -> usize { + self.0.control_context_instruction_pointer() + } + + /// Returns the frame pointer stored in the Fiber's ControlContext. + pub fn control_context_frame_pointer(&self) -> usize { + self.0.control_context_frame_pointer() + } + + /// Returns the stack pointer stored in the Fiber's ControlContext. + pub fn control_context_stack_pointer(&self) -> usize { + self.0.control_context_stack_pointer() + } + + /// Initializes this stack, such that it will execute the function denoted + /// by `func_ref`. `parameter_count` and `return_value_count` must be the + /// corresponding number of parameters and return values of `func_ref`. + /// `args` must point to the `args` field of the `VMContRef` owning this pointer. + /// + /// It will be updated by this function to correctly describe + /// the buffer used by this function for its arguments and return values. + pub fn initialize( + &self, + func_ref: *const VMFuncRef, + caller_vmctx: *mut VMContext, + args: *mut VMArray, + parameter_count: u32, + return_value_count: u32, + ) { + self.0.initialize( + func_ref, + caller_vmctx, + args, + parameter_count, + return_value_count, + ) + } +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs new file mode 100644 index 000000000000..e2042e352c19 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs @@ -0,0 +1,75 @@ +use anyhow::Result; +use core::ops::Range; + +use crate::runtime::vm::stack_switching::VMArray; +use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; + +#[allow(dead_code)] +#[derive(Debug, PartialEq, Eq)] +pub enum Allocator { + Mmap, + Custom, +} + +/// Making sure that this has the same size as the non-dummy version, to +/// make some tests happy. +#[derive(Debug)] +#[repr(C)] +pub struct VMContinuationStack { + _top: *mut u8, + _len: usize, + _allocator: Allocator, +} + +impl VMContinuationStack { + pub fn new(_size: usize) -> Result { + anyhow::bail!("Stack switching disabled or not implemented on this platform") + } + + pub fn unallocated() -> Self { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn is_unallocated(&self) -> bool { + panic!("Stack switching disabled or not implemented on this platform") + } + + #[allow(clippy::missing_safety_doc)] + pub unsafe fn from_raw_parts(_base: *mut u8, _guard_size: usize, _len: usize) -> Result { + anyhow::bail!("Stack switching disabled or not implemented on this platform") + } + + pub fn is_from_raw_parts(&self) -> bool { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn top(&self) -> Option<*mut u8> { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn range(&self) -> Option> { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn control_context_instruction_pointer(&self) -> usize { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn control_context_frame_pointer(&self) -> usize { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn control_context_stack_pointer(&self) -> usize { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn initialize( + &self, + _func_ref: *const VMFuncRef, + _caller_vmctx: *mut VMContext, + _args: *mut VMArray, + _parameter_count: u32, + _return_value_count: u32, + ) { + } +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs new file mode 100644 index 000000000000..e50e1f2c7fbe --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs @@ -0,0 +1,354 @@ +//! The stack layout is expected to look like so: +//! +//! +//! ```text +//! 0xB000 +-----------------------+ <- top of stack (TOS) +//! | saved RIP | +//! 0xAff8 +-----------------------+ +//! | saved RBP | +//! 0xAff0 +-----------------------+ +//! | saved RSP | +//! 0xAfe8 +-----------------------+ <- beginning of "control context", +//! | args_capacity | +//! 0xAfe0 +-----------------------+ +//! | args buffer, size: | +//! | (16 * args_capacity) | +//! 0xAfc0 +-----------------------+ <- below: beginning of usable stack space +//! | | (16-byte aligned) +//! | | +//! ~ ... ~ <- actual native stack space to use +//! | | +//! 0x1000 +-----------------------+ +//! | guard page | <- (not currently enabled) +//! 0x0000 +-----------------------+ +//! ``` +//! +//! The "control context" indicates how to resume a computation. The layout is +//! determined by Cranelift's stack_switch instruction, which reads and writes +//! these fields. The fields are used as follows, where we distinguish two +//! cases: +//! +//! 1. +//! If the continuation is currently active (i.e., running directly, or ancestor +//! of the running continuation), it stores the PC, RSP, and RBP of the *parent* +//! of the running continuation. +//! +//! 2. +//! If the picture shows a suspended computation, the fields store the PC, RSP, +//! and RBP at the time of the suspension. +//! +//! Note that this design ensures that external tools can construct backtraces +//! in the presence of stack switching by using frame pointers only: The +//! wasmtime_continuation_start trampoline uses the address of the RBP field in the +//! control context (0xAff0 above) as its frame pointer. This means that when +//! passing the wasmtime_continuation_start frame while doing frame pointer walking, +//! the parent of that frame is the last frame in the parent of this +//! continuation. +//! +//! Wasmtime's own mechanism for constructing backtraces also relies on frame +//! pointer chains. However, it understands continuations and does not rely on +//! the trickery outlined here to go from the frames in one continuation to the +//! parent. +//! +//! The args buffer is used as follows: It is used by the array calling +//! trampoline to read and store the arguments and return values of the function +//! running inside the continuation. If this function has m parameters and n +//! return values, then args_capacity is defined as max(m, n) and the size of +//! the args buffer is args_capacity * 16 bytes. The start address (0xAfc0 in +//! the example above, thus assuming args_capacity = 2) is saved as the `data` +//! field of the VMContRef's `args` object. + +#![allow(unused_macros)] + +use core::ptr::NonNull; +use std::io; +use std::ops::Range; +use std::ptr; + +use crate::runtime::vm::stack_switching::VMArray; +use crate::runtime::vm::{VMContext, VMFuncRef, VMOpaqueContext, ValRaw}; + +#[derive(Debug, PartialEq, Eq)] +pub enum Allocator { + Mmap, + Custom, +} + +#[derive(Debug)] +#[repr(C)] +pub struct VMContinuationStack { + // The top of the stack; for stacks allocated by the fiber implementation itself, + // the base address of the allocation will be `top.sub(len.unwrap())` + top: *mut u8, + // The length of the stack + len: usize, + // allocation strategy + allocator: Allocator, +} + +impl VMContinuationStack { + pub fn new(size: usize) -> io::Result { + // Round up our stack size request to the nearest multiple of the + // page size. + let page_size = rustix::param::page_size(); + let size = if size == 0 { + page_size + } else { + (size + (page_size - 1)) & (!(page_size - 1)) + }; + + unsafe { + // Add in one page for a guard page and then ask for some memory. + let mmap_len = size + page_size; + let mmap = rustix::mm::mmap_anonymous( + ptr::null_mut(), + mmap_len, + rustix::mm::ProtFlags::empty(), + rustix::mm::MapFlags::PRIVATE, + )?; + + rustix::mm::mprotect( + mmap.cast::().add(page_size).cast(), + size, + rustix::mm::MprotectFlags::READ | rustix::mm::MprotectFlags::WRITE, + )?; + + Ok(Self { + top: mmap.cast::().add(mmap_len), + len: mmap_len, + allocator: Allocator::Mmap, + }) + } + } + + pub fn unallocated() -> Self { + Self { + top: std::ptr::null_mut(), + len: 0, + allocator: Allocator::Custom, + } + } + + pub fn is_unallocated(&self) -> bool { + debug_assert_eq!(self.len == 0, self.top == std::ptr::null_mut()); + self.len == 0 + } + + #[allow(clippy::missing_safety_doc)] + pub unsafe fn from_raw_parts( + base: *mut u8, + _guard_size: usize, + len: usize, + ) -> io::Result { + Ok(Self { + top: base.add(len), + len, + allocator: Allocator::Custom, + }) + } + + pub fn is_from_raw_parts(&self) -> bool { + self.allocator == Allocator::Custom + } + + pub fn top(&self) -> Option<*mut u8> { + Some(self.top) + } + + pub fn range(&self) -> Option> { + let base = unsafe { self.top.sub(self.len) as usize }; + Some(base..base + self.len) + } + + pub fn control_context_instruction_pointer(&self) -> usize { + // See picture at top of this file: + // RIP is stored 8 bytes below top of stack. + unsafe { + let ptr = self.top.sub(8) as *mut usize; + *ptr + } + } + + pub fn control_context_frame_pointer(&self) -> usize { + // See picture at top of this file: + // RBP is stored 16 bytes below top of stack. + unsafe { + let ptr = self.top.sub(16) as *mut usize; + *ptr + } + } + + pub fn control_context_stack_pointer(&self) -> usize { + // See picture at top of this file: + // RSP is stored 24 bytes below top of stack. + unsafe { + let ptr = self.top.sub(24) as *mut usize; + *ptr + } + } + + /// This function installs the launchpad for the computation to run on the + /// fiber, such that executing a `stack_switch` instruction on the stack + /// actually runs the desired computation. + /// + /// Concretely, switching to the stack prepared by this function + /// causes that we enter `wasmtime_continuation_start`, which then in turn + /// calls `fiber_start` with the following arguments: + /// TOS, func_ref, caller_vmctx, args_ptr, args_capacity + /// + /// Note that at this point we also allocate the args buffer + /// (see picture at the top of this file). + /// We define `args_capacity` as the max of parameter and return value count. + /// Then the size s of the actual buffer size is calculated as follows: + /// s = size_of(ValRaw) * `args_capacity`, + /// + /// Note that this value is used below, and we may have s = 0. + /// + /// The layout of the VMContinuationStack near the top of stack (TOS) + /// *after* running this function is as follows: + /// + /// + /// Offset from | + /// TOS | Contents + /// ---------------|------------------------------------------------------- + /// -0x08 | address of wasmtime_continuation_start function (future PC) + /// -0x10 | TOS - 0x10 (future RBP) + /// -0x18 | TOS - 0x40 - s (future RSP) + /// -0x20 | args_capacity + /// + /// + /// The data stored behind the args buffer is as follows: + /// + /// Offset from | + /// TOS | Contents + /// ---------------|------------------------------------------------------- + /// -0x28 - s | func_ref + /// -0x30 - s | caller_vmctx + /// -0x38 - s | args (of type *mut ArrayRef) + /// -0x40 - s | return_value_count + pub fn initialize( + &self, + func_ref: *const VMFuncRef, + caller_vmctx: *mut VMContext, + args: *mut VMArray, + parameter_count: u32, + return_value_count: u32, + ) { + let tos = self.top; + + unsafe { + let store = |tos_neg_offset, value| { + let target = tos.sub(tos_neg_offset) as *mut usize; + target.write(value) + }; + + let args = &mut *args; + let args_capacity = std::cmp::max(parameter_count, return_value_count); + // The args object must currently be empty. + debug_assert_eq!(args.capacity, 0); + debug_assert_eq!(args.length, 0); + + // The size of the args buffer + let s = args_capacity as usize * std::mem::size_of::(); + + // The actual pointer to the buffer + let args_data_ptr = if args_capacity == 0 { + 0 + } else { + tos as usize - 0x20 - s + }; + + args.capacity = args_capacity; + args.data = args_data_ptr as *mut ValRaw; + + let to_store = [ + // Data near top of stack: + (0x08, wasmtime_continuation_start as usize), + (0x10, tos.sub(0x10) as usize), + (0x18, tos.sub(0x40 + s) as usize), + (0x20, args_capacity as usize), + // Data after the args buffer: + (0x28 + s, func_ref as usize), + (0x30 + s, caller_vmctx as usize), + (0x38 + s, args as *mut VMArray as usize), + (0x40 + s, return_value_count as usize), + ]; + + for (offset, data) in to_store { + store(offset, data); + } + } + } +} + +impl Drop for VMContinuationStack { + fn drop(&mut self) { + unsafe { + match self.allocator { + Allocator::Mmap => { + let ret = rustix::mm::munmap(self.top.sub(self.len) as _, self.len); + debug_assert!(ret.is_ok()); + } + Allocator::Custom => {} // It's the creator's responsibility to reclaim the memory. + } + } + } +} + +unsafe extern "C" { + #[allow(dead_code)] // only used in inline assembly for some platforms + fn wasmtime_continuation_start(); +} + +/// This function is responsible for actually running a wasm function inside a +/// continuation. It is only ever called from `wasmtime_continuation_start`. Hence, it +/// must never return. +unsafe extern "C" fn fiber_start( + func_ref: *const VMFuncRef, + caller_vmctx: *mut VMContext, + args: *mut VMArray, + return_value_count: u32, +) { + unsafe { + let func_ref = func_ref.as_ref().expect("Non-null function reference"); + let caller_vmxtx = VMOpaqueContext::from_vmcontext(NonNull::new_unchecked(caller_vmctx)); + let args = &mut *args; + let params_and_returns: NonNull<[ValRaw]> = if args.capacity == 0 { + NonNull::from(&[]) + } else { + std::slice::from_raw_parts_mut(args.data, args.capacity as usize).into() + }; + + // NOTE(frank-emrich) The usage of the `caller_vmctx` is probably not + // 100% correct here. Currently, we determine the "caller" vmctx when + // initilizing the fiber stack/continuation (i.e. as part of + // `cont.new`). However, we may subsequenly `resume` the continuation + // from a different Wasm instance. The way to fix this would be to make + // the currently active `VMContext` an additional parameter of + // `wasmtime_continuation_switch` and pipe it through to this point. However, + // since the caller vmctx is only really used to access stuff in the + // underlying `Store`, it's fine to be slightly sloppy about the exact + // value we set. + func_ref.array_call(None, caller_vmxtx, params_and_returns); // TODO(dhil): we are ignoring the boolean return value + // here... we probably shouldn't. + + // The array call trampoline should have just written + // `return_value_count` values to the `args` buffer. Let's reflect that + // in its length field, to make various bounds checks happy. + args.length = return_value_count; + + // Note that after this function returns, wasmtime_continuation_start + // will switch back to the parent stack. + } +} + +cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + mod x86_64; + } else { + // Note that this shoul be unreachable: In stack.rs, we currently select + // the module defined in the current file only if we are on unix AND + // x86_64. + compile_error!("the stack switching feature is not supported on this CPU architecture"); + } +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs new file mode 100644 index 000000000000..61ffd66f68e6 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs @@ -0,0 +1,86 @@ +// A WORD OF CAUTION +// +// This entire file basically needs to be kept in sync with itself. It's not +// really possible to modify just one bit of this file without understanding +// all the other bits. Documentation tries to reference various bits here and +// there but try to make sure to read over everything before tweaking things! + +use wasmtime_asm_macros::asm_func; + +// This is a pretty special function that has no real signature. Its use is to +// be the "base" function of all fibers. This entrypoint is used in +// `wasmtime_continuation_init` to bootstrap the execution of a new fiber. +// +// We also use this function as a persistent frame on the stack to emit dwarf +// information to unwind into the caller. This allows us to unwind from the +// fiber's stack back to the initial stack that the fiber was called from. We use +// special dwarf directives here to do so since this is a pretty nonstandard +// function. +// +// If you're curious a decent introduction to CFI things and unwinding is at +// https://www.imperialviolet.org/2017/01/18/cfi.html +// +// Note that this function is never called directly. It is only ever entered +// when a `stack_switch` instruction loads its address when switching to a stack +// prepared by `FiberStack::initialize`. +// +// Executing `stack_switch` on a stack prepared by `FiberStack::initialize` as +// described in the comment on `FiberStack::initialize` leads to the following +// values in various registers when execution of wasmtime_continuation_start begins: +// +// RSP: TOS - 0x40 - (16 * `args_capacity`) +// RBP: TOS - 0x10 +asm_func!( + "wasmtime_continuation_start", + " + // TODO(frank-emrich): Restore DWARF information for this function. In + // the meantime, debugging is possible using frame pointer walking. + + + // + // Note that the next 4 instructions amount to calling fiber_start + // with the following arguments: + // 1. func_ref + // 2. caller_vmctx + // 3. args (of type *mut ArrayRef) + // 4. return_value_count + // + + pop rcx // return_value_count + pop rdx // args + pop rsi // caller_vmctx + pop rdi // func_ref + // Note that RBP already contains the right frame pointer to build a + // frame pointer chain including the parent continuation: + // The current value of RBP is where we store the parent RBP in the + // control context! + call {fiber_start} + + // Return to the parent continuation. + // RBP is callee-saved (no matter if it's used as a frame pointe or + // not), so its value is still TOS - 0x10. + // Use that fact to obtain saved parent RBP, RSP, and RIP from control + // context near TOS. + mov rsi, 0x08[rbp] // putting new RIP in temp register + mov rsp, -0x08[rbp] + mov rbp, [rbp] + + // The stack_switch instruction uses register RDI for the payload. + // Here, the payload indicates that we are returning (value 0). + // See the test case below to keep this in sync with + // ControlEffect::return_() + mov rdi, 0 + + jmp rsi + ", + fiber_start = sym super::fiber_start, +); + +#[test] +fn test_return_payload() { + // The following assumption is baked into `wasmtime_continuation_start`. + assert_eq!( + wasmtime_environ::stack_switching::CONTROL_EFFECT_RETURN_DISCRIMINANT, + 0 + ); +} diff --git a/crates/wasmtime/src/runtime/vm/table.rs b/crates/wasmtime/src/runtime/vm/table.rs index dc52dd32fead..c0e1515f008e 100644 --- a/crates/wasmtime/src/runtime/vm/table.rs +++ b/crates/wasmtime/src/runtime/vm/table.rs @@ -5,6 +5,7 @@ #![cfg_attr(feature = "gc", allow(irrefutable_let_patterns))] use crate::prelude::*; +use crate::runtime::vm::stack_switching::VMContObj; use crate::runtime::vm::vmcontext::{VMFuncRef, VMTableDefinition}; use crate::runtime::vm::{GcStore, SendSyncPtr, VMGcRef, VMStore}; use core::alloc::Layout; @@ -34,12 +35,16 @@ pub enum TableElement { /// (which has access to the info needed for lazy initialization) /// will replace it when fetched. UninitFunc, + + /// A `contref` + ContRef(Option), } #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum TableElementType { Func, GcRef, + Cont, } impl TableElementType { @@ -47,9 +52,19 @@ impl TableElementType { match (val, self) { (TableElement::FuncRef(_), TableElementType::Func) => true, (TableElement::GcRef(_), TableElementType::GcRef) => true, + (TableElement::ContRef(_), TableElementType::Cont) => true, _ => false, } } + + /// Returns the size required to actually store an element of this particular type + pub fn element_size(&self) -> usize { + match self { + TableElementType::Func => core::mem::size_of::(), + TableElementType::GcRef => core::mem::size_of::>(), + TableElementType::Cont => core::mem::size_of::(), + } + } } // The usage of `*mut VMFuncRef` is safe w.r.t. thread safety, this just relies @@ -74,6 +89,7 @@ impl TableElement { Self::FuncRef(e) => e, Self::UninitFunc => panic!("Uninitialized table element value outside of table slot"), Self::GcRef(_) => panic!("GC reference is not a function reference"), + Self::ContRef(_) => panic!("Continuation reference is not a function reference"), } } @@ -105,6 +121,18 @@ impl From for TableElement { } } +impl From> for TableElement { + fn from(c: Option) -> TableElement { + TableElement::ContRef(c) + } +} + +impl From for TableElement { + fn from(c: VMContObj) -> TableElement { + TableElement::ContRef(Some(c)) + } +} + #[derive(Copy, Clone)] #[repr(transparent)] struct TaggedFuncRef(*mut VMFuncRef); @@ -140,10 +168,40 @@ impl TaggedFuncRef { } pub type FuncTableElem = Option>; +pub type ContTableElem = Option; + +/// The maximum of the sizes of any of the table element types +#[allow(dead_code, reason = "Only used if pooling allocator is enabled")] +pub const MAX_TABLE_ELEM_SIZE: usize = { + let sizes = [ + core::mem::size_of::(), + core::mem::size_of::(), + core::mem::size_of::>(), + ]; + + // This is equivalent to `|data| {data.iter().reduce(std::cmp::max).unwrap()}`, + // but as a `const` function, so we can use it to define a constant. + const fn slice_max(data: &[usize]) -> usize { + match data { + [] => 0, + [head, tail @ ..] => { + let tail_max = slice_max(tail); + if *head >= tail_max { + *head + } else { + tail_max + } + } + } + } + + slice_max(&sizes) +}; pub enum StaticTable { Func(StaticFuncTable), GcRef(StaticGcRefTable), + Cont(StaticContTable), } impl From for StaticTable { @@ -158,6 +216,12 @@ impl From for StaticTable { } } +impl From for StaticTable { + fn from(value: StaticContTable) -> Self { + Self::Cont(value) + } +} + pub struct StaticFuncTable { /// Where data for this table is stored. The length of this list is the /// maximum size of the table. @@ -176,9 +240,18 @@ pub struct StaticGcRefTable { size: usize, } +pub struct StaticContTable { + /// Where data for this table is stored. The length of this list is the + /// maximum size of the table. + data: SendSyncPtr<[ContTableElem]>, + /// The current size of the table. + size: usize, +} + pub enum DynamicTable { Func(DynamicFuncTable), GcRef(DynamicGcRefTable), + Cont(DynamicContTable), } impl From for DynamicTable { @@ -193,6 +266,12 @@ impl From for DynamicTable { } } +impl From for DynamicTable { + fn from(value: DynamicContTable) -> Self { + Self::Cont(value) + } +} + pub struct DynamicFuncTable { /// Dynamically managed storage space for this table. The length of this /// vector is the current size of the table. @@ -211,6 +290,14 @@ pub struct DynamicGcRefTable { maximum: Option, } +pub struct DynamicContTable { + /// Dynamically managed storage space for this table. The length of this + /// vector is the current size of the table. + elements: Vec, + /// Maximum size that `elements` can grow to. + maximum: Option, +} + /// Represents an instance's table. pub enum Table { /// A "static" table where storage space is managed externally, currently @@ -241,6 +328,13 @@ impl From for Table { } } +impl From for Table { + fn from(value: StaticContTable) -> Self { + let t: StaticTable = value.into(); + t.into() + } +} + impl From for Table { fn from(value: DynamicTable) -> Self { Self::Dynamic(value) @@ -261,11 +355,18 @@ impl From for Table { } } -fn wasm_to_table_type(ty: WasmRefType) -> TableElementType { +impl From for Table { + fn from(value: DynamicContTable) -> Self { + let t: DynamicTable = value.into(); + t.into() + } +} + +pub(crate) fn wasm_to_table_type(ty: WasmRefType) -> TableElementType { match ty.heap_type.top() { WasmHeapTopType::Func => TableElementType::Func, WasmHeapTopType::Any | WasmHeapTopType::Extern => TableElementType::GcRef, - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => TableElementType::Cont, } } @@ -326,6 +427,10 @@ impl Table { elements: unsafe { alloc_dynamic_table_elements(minimum)? }, maximum, })), + TableElementType::Cont => Ok(Self::from(DynamicContTable { + elements: vec![None; minimum], + maximum: maximum, + })), } } @@ -385,6 +490,26 @@ impl Table { )); Ok(Self::from(StaticGcRefTable { data, size })) } + TableElementType::Cont => { + let len = { + let data = data.as_non_null().as_ref(); + let (before, data, after) = data.align_to::(); + assert!(before.is_empty()); + assert!(after.is_empty()); + data.len() + }; + ensure!( + usize::try_from(ty.limits.min).unwrap() <= len, + "initial table size of {} exceeds the pooling allocator's \ + configured maximum table size of {len} elements", + ty.limits.min, + ); + let data = SendSyncPtr::new(NonNull::slice_from_raw_parts( + data.as_non_null().cast::(), + cmp::min(len, max), + )); + Ok(Self::from(StaticContTable { data, size })) + } } } @@ -441,6 +566,9 @@ impl Table { Table::Static(StaticTable::GcRef(_)) | Table::Dynamic(DynamicTable::GcRef(_)) => { TableElementType::GcRef } + Table::Static(StaticTable::Cont(_)) | Table::Dynamic(DynamicTable::Cont(_)) => { + TableElementType::Cont + } } } @@ -455,10 +583,12 @@ impl Table { match self { Table::Static(StaticTable::Func(StaticFuncTable { size, .. })) => *size, Table::Static(StaticTable::GcRef(StaticGcRefTable { size, .. })) => *size, + Table::Static(StaticTable::Cont(StaticContTable { size, .. })) => *size, Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => elements.len(), Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => { elements.len() } + Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => elements.len(), } } @@ -470,10 +600,12 @@ impl Table { /// when it is being constrained by an instance allocator. pub fn maximum(&self) -> Option { match self { + Table::Static(StaticTable::Cont(StaticContTable { data, .. })) => Some(data.len()), Table::Static(StaticTable::Func(StaticFuncTable { data, .. })) => Some(data.len()), Table::Static(StaticTable::GcRef(StaticGcRefTable { data, .. })) => Some(data.len()), Table::Dynamic(DynamicTable::Func(DynamicFuncTable { maximum, .. })) => *maximum, Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { maximum, .. })) => *maximum, + Table::Dynamic(DynamicTable::Cont(DynamicContTable { maximum, .. })) => *maximum, } } @@ -572,6 +704,10 @@ impl Table { let (funcrefs, _lazy_init) = self.funcrefs_mut(); funcrefs[start..end].fill(TaggedFuncRef::UNINIT); } + TableElement::ContRef(c) => { + let contrefs = self.contrefs_mut(); + contrefs[start..end].fill(c); + } } Ok(()) @@ -652,6 +788,12 @@ impl Table { } *size = new_size; } + Table::Static(StaticTable::Cont(StaticContTable { data, size })) => { + unsafe { + debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none())); + } + *size = new_size; + } // These calls to `resize` could move the base address of // `elements`. If this table's limits declare it to be fixed-size, @@ -666,6 +808,9 @@ impl Table { Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => { elements.resize_with(usize::try_from(new_size).unwrap(), || None); } + Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => { + elements.resize(usize::try_from(new_size).unwrap(), None); + } } self.fill( @@ -698,6 +843,11 @@ impl Table { let r = r.as_ref().map(|r| gc_store.unwrap().clone_gc_ref(r)); TableElement::GcRef(r) }), + TableElementType::Cont => self + .contrefs() + .get(index) + .copied() + .map(|e| TableElement::ContRef(e)), } } @@ -725,6 +875,9 @@ impl Table { TableElement::GcRef(e) => { *self.gc_refs_mut().get_mut(index).ok_or(())? = e; } + TableElement::ContRef(c) => { + *self.contrefs_mut().get_mut(index).ok_or(())? = c; + } } Ok(()) } @@ -792,6 +945,10 @@ impl Table { current_elements: *size, } } + Table::Static(StaticTable::Cont(StaticContTable { data, size })) => VMTableDefinition { + base: data.cast().into(), + current_elements: *size, + }, Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => { VMTableDefinition { base: NonNull::<[FuncTableElem]>::from(&mut elements[..]) @@ -808,6 +965,14 @@ impl Table { current_elements: elements.len(), } } + Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => { + VMTableDefinition { + base: NonNull::<[Option]>::from(&mut elements[..]) + .cast() + .into(), + current_elements: elements.len().try_into().unwrap(), + } + } } } @@ -876,6 +1041,32 @@ impl Table { } } + fn contrefs(&self) -> &[Option] { + assert_eq!(self.element_type(), TableElementType::Cont); + match self { + Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe { + slice::from_raw_parts(elements.as_ptr().cast(), elements.len()) + }, + Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe { + slice::from_raw_parts(data.as_ptr().cast(), usize::try_from(*size).unwrap()) + }, + _ => unreachable!(), + } + } + + fn contrefs_mut(&mut self) -> &mut [Option] { + assert_eq!(self.element_type(), TableElementType::Cont); + match self { + Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe { + slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len()) + }, + Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe { + slice::from_raw_parts_mut(data.as_ptr().cast(), usize::try_from(*size).unwrap()) + }, + _ => unreachable!(), + } + } + /// Get this table's GC references as a slice. /// /// Panics if this is not a table of GC references. @@ -924,6 +1115,11 @@ impl Table { ); } } + TableElementType::Cont => { + // `contref` are `Copy`, so just do a mempcy + dst_table.contrefs_mut()[dst_range] + .copy_from_slice(&src_table.contrefs()[src_range]); + } } } @@ -972,6 +1168,10 @@ impl Table { } } } + TableElementType::Cont => { + // `contref` are `Copy`, so just do a memmove + self.contrefs_mut().copy_within(src_range, dst_range.start); + } } } } diff --git a/crates/wasmtime/src/runtime/vm/traphandlers.rs b/crates/wasmtime/src/runtime/vm/traphandlers.rs index 62da746f404c..019a56c5dab7 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers.rs @@ -20,12 +20,15 @@ use crate::runtime::module::lookup_code; use crate::runtime::store::{ExecutorRef, StoreOpaque}; use crate::runtime::vm::sys::traphandlers; use crate::runtime::vm::{Instance, InterpreterRef, VMContext, VMStoreContext}; -use crate::{StoreContextMut, WasmBacktrace}; +use crate::{EntryStoreContext, StoreContextMut, WasmBacktrace}; use core::cell::Cell; use core::ops::Range; use core::ptr::{self, NonNull}; pub use self::backtrace::Backtrace; +#[cfg(feature = "gc")] +pub use self::backtrace::Frame; + pub use self::coredump::CoreDumpStack; pub use self::tls::tls_eager_initialize; #[cfg(feature = "async")] @@ -356,46 +359,51 @@ impl From for TrapReason { /// longjmp'd over and none of its destructors on the stack may be run. pub unsafe fn catch_traps( store: &mut StoreContextMut<'_, T>, + old_state: &EntryStoreContext, mut closure: F, ) -> Result<(), Box> where F: FnMut(NonNull, Option>) -> bool, { let caller = store.0.default_caller(); - let result = CallThreadState::new(store.0, caller).with(|cx| match store.0.executor() { - // In interpreted mode directly invoke the host closure since we won't - // be using host-based `setjmp`/`longjmp` as that's not going to save - // the context we want. - ExecutorRef::Interpreter(r) => { - cx.jmp_buf - .set(CallThreadState::JMP_BUF_INTERPRETER_SENTINEL); - closure(caller, Some(r)) - } + let result = + CallThreadState::new(store.0, caller, old_state).with(|cx| match store.0.executor() { + // In interpreted mode directly invoke the host closure since we won't + // be using host-based `setjmp`/`longjmp` as that's not going to save + // the context we want. + ExecutorRef::Interpreter(r) => { + cx.jmp_buf + .set(CallThreadState::JMP_BUF_INTERPRETER_SENTINEL); + closure(caller, Some(r)) + } - // In native mode, however, defer to C to do the `setjmp` since Rust - // doesn't understand `setjmp`. - // - // Note that here we pass a function pointer to C to catch longjmp - // within, here it's `call_closure`, and that passes `None` for the - // interpreter since this branch is only ever taken if the interpreter - // isn't present. - #[cfg(has_host_compiler_backend)] - ExecutorRef::Native => traphandlers::wasmtime_setjmp( - cx.jmp_buf.as_ptr(), - { - extern "C" fn call_closure(payload: *mut u8, caller: NonNull) -> bool - where - F: FnMut(NonNull, Option>) -> bool, + // In native mode, however, defer to C to do the `setjmp` since Rust + // doesn't understand `setjmp`. + // + // Note that here we pass a function pointer to C to catch longjmp + // within, here it's `call_closure`, and that passes `None` for the + // interpreter since this branch is only ever taken if the interpreter + // isn't present. + #[cfg(has_host_compiler_backend)] + ExecutorRef::Native => traphandlers::wasmtime_setjmp( + cx.jmp_buf.as_ptr(), { - unsafe { (*(payload as *mut F))(caller, None) } - } - - call_closure:: - }, - &mut closure as *mut F as *mut u8, - caller, - ), - }); + extern "C" fn call_closure( + payload: *mut u8, + caller: NonNull, + ) -> bool + where + F: FnMut(NonNull, Option>) -> bool, + { + unsafe { (*(payload as *mut F))(caller, None) } + } + + call_closure:: + }, + &mut closure as *mut F as *mut u8, + caller, + ), + }); return match result { Ok(x) => Ok(x), @@ -413,7 +421,9 @@ where // usage of its accessor methods. mod call_thread_state { use super::*; + use crate::runtime::vm::stack_switching::VMStackChain; use crate::runtime::vm::Unwind; + use crate::EntryStoreContext; /// Temporary state stored on the stack which is registered in the `tls` module /// below for calls into wasm. @@ -433,17 +443,16 @@ mod call_thread_state { #[cfg(all(has_native_signals, unix))] pub(crate) async_guard_range: Range<*mut u8>, - // The values of `VMStoreContext::last_wasm_{exit_{pc,fp},entry_sp}` for - // the *previous* `CallThreadState` for this same store/limits. Our - // *current* last wasm PC/FP/SP are saved in `self.vm_store_context`. We - // save a copy of the old registers here because the `VMStoreContext` - // typically doesn't change across nested calls into Wasm (i.e. they are - // typically calls back into the same store and `self.vm_store_context - // == self.prev.vm_store_context`) and we must to maintain the list of - // contiguous-Wasm-frames stack regions for backtracing purposes. - old_last_wasm_exit_fp: Cell, - old_last_wasm_exit_pc: Cell, - old_last_wasm_entry_fp: Cell, + // The state of the runtime for the *previous* `CallThreadState` for + // this same store. Our *current* state is saved in `self.limits`, + // `self.stack_chain`, etc. We need access to the old values of these + // fields because the `VMStoreContext` typically doesn't change across + // nested calls into Wasm (i.e. they are typically calls back into the + // same store and `self.vm_store_context == self.prev.vm_store_context`) and we must to + // maintain the list of contiguous-Wasm-frames stack regions for + // backtracing purposes. + // FIXME(frank-emrich) Does this need to be an (Unsafe)Cell? + old_state: *const EntryStoreContext, } impl Drop for CallThreadState { @@ -451,13 +460,6 @@ mod call_thread_state { // Unwind information should not be present as it should have // already been processed. debug_assert!(self.unwind.replace(None).is_none()); - - unsafe { - let cx = self.vm_store_context.as_ref(); - *cx.last_wasm_exit_fp.get() = self.old_last_wasm_exit_fp.get(); - *cx.last_wasm_exit_pc.get() = self.old_last_wasm_exit_pc.get(); - *cx.last_wasm_entry_fp.get() = self.old_last_wasm_entry_fp.get(); - } } } @@ -465,7 +467,11 @@ mod call_thread_state { pub const JMP_BUF_INTERPRETER_SENTINEL: *mut u8 = 1 as *mut u8; #[inline] - pub(super) fn new(store: &mut StoreOpaque, caller: NonNull) -> CallThreadState { + pub(super) fn new( + store: &mut StoreOpaque, + caller: NonNull, + old_state: *const EntryStoreContext, + ) -> CallThreadState { let vm_store_context = unsafe { Instance::from_vmctx(caller, |i| i.vm_store_context()) .read() @@ -490,31 +496,28 @@ mod call_thread_state { #[cfg(all(has_native_signals, unix))] async_guard_range: store.async_guard_range(), prev: Cell::new(ptr::null()), - old_last_wasm_exit_fp: Cell::new(unsafe { - *vm_store_context.as_ref().last_wasm_exit_fp.get() - }), - old_last_wasm_exit_pc: Cell::new(unsafe { - *vm_store_context.as_ref().last_wasm_exit_pc.get() - }), - old_last_wasm_entry_fp: Cell::new(unsafe { - *vm_store_context.as_ref().last_wasm_entry_fp.get() - }), + old_state, } } /// Get the saved FP upon exit from Wasm for the previous `CallThreadState`. - pub fn old_last_wasm_exit_fp(&self) -> usize { - self.old_last_wasm_exit_fp.get() + pub unsafe fn old_last_wasm_exit_fp(&self) -> usize { + (&*self.old_state).last_wasm_exit_fp } /// Get the saved PC upon exit from Wasm for the previous `CallThreadState`. - pub fn old_last_wasm_exit_pc(&self) -> usize { - self.old_last_wasm_exit_pc.get() + pub unsafe fn old_last_wasm_exit_pc(&self) -> usize { + (&*self.old_state).last_wasm_exit_pc } /// Get the saved FP upon entry into Wasm for the previous `CallThreadState`. - pub fn old_last_wasm_entry_fp(&self) -> usize { - self.old_last_wasm_entry_fp.get() + pub unsafe fn old_last_wasm_entry_fp(&self) -> usize { + (&*self.old_state).last_wasm_entry_fp + } + + /// Get the saved `VMStackChain` for the previous `CallThreadState`. + pub unsafe fn old_stack_chain(&self) -> VMStackChain { + (&*self.old_state).stack_chain.clone() } /// Get the previous `CallThreadState`. diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index 77fbfe58b534..c147097eff3b 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -23,10 +23,12 @@ use crate::prelude::*; use crate::runtime::store::StoreOpaque; +use crate::runtime::vm::stack_switching::VMStackChain; use crate::runtime::vm::{ traphandlers::{tls, CallThreadState}, Unwind, VMStoreContext, }; +use crate::vm::stack_switching::{VMContRef, VMStackState}; use core::ops::ControlFlow; /// A WebAssembly stack trace. @@ -107,6 +109,38 @@ impl Backtrace { }); } + // Walk the stack of the given continuation, which must be suspended, and + // all of its parent continuations (if any). + #[allow(dead_code, reason = "Only used by GC code at the moment")] + pub fn trace_suspended_continuation( + store: &StoreOpaque, + continuation: &VMContRef, + f: impl FnMut(Frame) -> ControlFlow<()>, + ) { + assert_eq!( + continuation.common_stack_information.state, + VMStackState::Suspended + ); + + let unwind = store.unwinder(); + + let pc = continuation.stack.control_context_instruction_pointer(); + let fp = continuation.stack.control_context_frame_pointer(); + let trampoline_fp = continuation + .common_stack_information + .limits + .last_wasm_entry_fp; + + unsafe { + // FIXME(frank-emrich) Casting from *const to *mut pointer is + // terrible, but we won't actually modify any of the continuations + // here. + let stack_chain = + VMStackChain::Continuation(continuation as *const VMContRef as *mut VMContRef); + Self::trace_through_continuations(unwind, stack_chain, pc, fp, trampoline_fp, f); + } + } + /// Walk the current Wasm stack, calling `f` for each frame we walk. /// /// If Wasm hit a trap, and we calling this from the trap handler, then the @@ -141,7 +175,17 @@ impl Backtrace { } }; + let stack_chain = (*(*vm_store_context).stack_chain.get()).clone(); + + // The first value in `activations` is for the most recently running + // wasm. We thus provide the stack chain of `first_wasm_state` to + // traverse the potential continuation stacks. For the subsequent + // activations, we unconditionally use `None` as the corresponding stack + // chain. This is justified because only the most recent execution of + // wasm may execute off the initial stack (see comments in + // `wasmtime::invoke_wasm_and_catch_traps` for details). let activations = core::iter::once(( + stack_chain, last_wasm_exit_pc, last_wasm_exit_fp, *(*vm_store_context).last_wasm_entry_fp.get(), @@ -149,25 +193,31 @@ impl Backtrace { .chain( state .iter() + .flat_map(|state| state.iter()) .filter(|state| core::ptr::eq(vm_store_context, state.vm_store_context.as_ptr())) .map(|state| { ( + state.old_stack_chain(), state.old_last_wasm_exit_pc(), state.old_last_wasm_exit_fp(), state.old_last_wasm_entry_fp(), ) }), ) - .take_while(|&(pc, fp, sp)| { - if pc == 0 { - debug_assert_eq!(fp, 0); - debug_assert_eq!(sp, 0); + .take_while(|(chain, pc, fp, sp)| { + if *pc == 0 { + debug_assert_eq!(*fp, 0); + debug_assert_eq!(*sp, 0); + } else { + debug_assert_ne!(chain.clone(), VMStackChain::Absent) } - pc != 0 + *pc != 0 }); - for (pc, fp, sp) in activations { - if let ControlFlow::Break(()) = Self::trace_through_wasm(unwind, pc, fp, sp, &mut f) { + for (chain, pc, fp, sp) in activations { + if let ControlFlow::Break(()) = + Self::trace_through_continuations(unwind, chain, pc, fp, sp, &mut f) + { log::trace!("====== Done Capturing Backtrace (closure break) ======"); return; } @@ -176,6 +226,100 @@ impl Backtrace { log::trace!("====== Done Capturing Backtrace (reached end of activations) ======"); } + /// Traces through a sequence of stacks, creating a backtrace for each one, + /// beginning at the given `pc` and `fp`. + /// + /// If `chain` is `InitialStack`, we are tracing through the initial stack, + /// and this function behaves like `trace_through_wasm`. + /// Otherwise, we can interpret `chain` as a linked list of stacks, which + /// ends with the initial stack. We then trace through each of these stacks + /// individually, up to (and including) the initial stack. + unsafe fn trace_through_continuations( + unwind: &dyn Unwind, + chain: VMStackChain, + pc: usize, + fp: usize, + trampoline_fp: usize, + mut f: impl FnMut(Frame) -> ControlFlow<()>, + ) -> ControlFlow<()> { + use crate::runtime::vm::stack_switching::{VMContRef, VMStackLimits}; + + // Handle the stack that is currently running (which may be a + // continuation or the initial stack). + Self::trace_through_wasm(unwind, pc, fp, trampoline_fp, &mut f)?; + + // Note that the rest of this function has no effect if `chain` is + // `Some(VMStackChain::InitialStack(_))` (i.e., there is only one stack to + // trace through: the initial stack) + + assert_ne!(chain, VMStackChain::Absent); + let stack_limits_vec: Vec<*mut VMStackLimits> = + chain.clone().into_stack_limits_iter().collect(); + let continuations_vec: Vec<*mut VMContRef> = + chain.clone().into_continuation_iter().collect(); + + // The VMStackLimits of the currently running stack (whether that's a + // continuation or the initial stack) contains undefined data, the + // information about that stack is saved in the Store's + // `VMRuntimeLimits` and handled at the top of this function + // already. That's why we ignore `stack_limits_vec[0]`. + // + // Note that a continuation stack's control context stores + // information about how to resume execution *in its parent*. Thus, + // we combine the information from continuations_vec[i] with + // stack_limits_vec[i + 1] below to get information about a + // particular stack. + // + // There must be exactly one more `VMStackLimits` object than there + // are continuations, due to the initial stack having one, too. + assert_eq!(stack_limits_vec.len(), continuations_vec.len() + 1); + + for i in 0..continuations_vec.len() { + let (continuation, parent_continuation, parent_limits) = unsafe { + // The continuation whose control context we want to + // access, to get information about how to continue + // execution in its parent. + let continuation = &*continuations_vec[i]; + + // The stack limits describing the parent of `continuation`. + let parent_limits = &*stack_limits_vec[i + 1]; + + // The parent of `continuation`, if the parent is itself a + // continuation. Otherwise, if `continuation` is the last + // continuation (i.e., its parent is the initial stack), this is + // None. + let parent_continuation = if i + 1 < continuations_vec.len() { + Some(&*continuations_vec[i + 1]) + } else { + None + }; + (continuation, parent_continuation, parent_limits) + }; + let fiber_stack = continuation.fiber_stack(); + let resume_pc = fiber_stack.control_context_instruction_pointer(); + let resume_fp = fiber_stack.control_context_frame_pointer(); + + // If the parent is indeed a continuation, we know the + // boundaries of its stack and can perform some extra debugging + // checks. + let parent_stack_range = parent_continuation.and_then(|p| p.fiber_stack().range()); + parent_stack_range.inspect(|parent_stack_range| { + debug_assert!(parent_stack_range.contains(&resume_fp)); + debug_assert!(parent_stack_range.contains(&parent_limits.last_wasm_entry_fp)); + debug_assert!(parent_stack_range.contains(&parent_limits.stack_limit)); + }); + + Self::trace_through_wasm( + unwind, + resume_pc, + resume_fp, + parent_limits.last_wasm_entry_fp, + &mut f, + )? + } + ControlFlow::Continue(()) + } + /// Walk through a contiguous sequence of Wasm frames starting with the /// frame at the given PC and FP and ending at `trampoline_sp`. unsafe fn trace_through_wasm( diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index 9af40e983f5f..83f27fd9142b 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -7,6 +7,7 @@ pub use self::vm_host_func_context::VMArrayCallHostFuncContext; use crate::prelude::*; use crate::runtime::vm::{GcStore, InterpreterRef, VMGcRef, VmPtr, VmSafe}; use crate::store::StoreOpaque; +use crate::vm::stack_switching::VMStackChain; use core::cell::UnsafeCell; use core::ffi::c_void; use core::fmt; @@ -500,7 +501,7 @@ impl VMGlobalDefinition { global.init_gc_ref(store.gc_store_mut()?, r.as_ref()) } WasmHeapTopType::Func => *global.as_func_ref_mut() = raw.get_funcref().cast(), - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => *global.as_func_ref_mut() = raw.get_funcref().cast(), // TODO(#10248): temporary hack. }, } Ok(global) @@ -1051,6 +1052,10 @@ pub struct VMStoreContext { /// Used to find the end of a contiguous sequence of Wasm frames when /// walking the stack. pub last_wasm_entry_fp: UnsafeCell, + + /// Stack information used by stack switching instructions. See documentation + /// on `VMStackChain` for details. + pub stack_chain: UnsafeCell, } // The `VMStoreContext` type is a pod-type with no destructor, and we don't @@ -1072,6 +1077,7 @@ impl Default for VMStoreContext { last_wasm_exit_fp: UnsafeCell::new(0), last_wasm_exit_pc: UnsafeCell::new(0), last_wasm_entry_fp: UnsafeCell::new(0), + stack_chain: UnsafeCell::new(VMStackChain::Absent), } } } diff --git a/crates/wast-util/src/lib.rs b/crates/wast-util/src/lib.rs index 7fac6a30cb9a..70d5aa7310d1 100644 --- a/crates/wast-util/src/lib.rs +++ b/crates/wast-util/src/lib.rs @@ -187,6 +187,7 @@ macro_rules! foreach_config_option { component_model_async simd gc_types + stack_switching } }; } @@ -291,6 +292,7 @@ impl Compiler { || config.gc() || config.relaxed_simd() || config.gc_types() + || config.stack_switching() { return true; } @@ -303,6 +305,11 @@ impl Compiler { if config.threads() { return true; } + + // Stack switching is not supported by Pulley. + if config.stack_switching() { + return true; + } } } diff --git a/tests/all/main.rs b/tests/all/main.rs index 1922c1393a8d..d786421de239 100644 --- a/tests/all/main.rs +++ b/tests/all/main.rs @@ -42,6 +42,7 @@ mod stack_overflow; mod store; mod structs; mod table; +#[cfg(all(feature = "stack-switching", unix, target_arch = "x86_64"))] mod tags; mod threads; mod traps; From 175d1898c7966b7f9832cdbc2cd2c824389b4d44 Mon Sep 17 00:00:00 2001 From: Frank Emrich Date: Wed, 12 Mar 2025 20:20:34 +0000 Subject: [PATCH 2/6] prtest:full From f8162ae5a6654109512ee38f564fc5e4a0d6aeb5 Mon Sep 17 00:00:00 2001 From: Frank Emrich Date: Mon, 21 Apr 2025 22:20:41 +0100 Subject: [PATCH 3/6] make sure to use ControlFlow result in trace_suspended_continuation --- .../src/runtime/vm/traphandlers/backtrace.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index c147097eff3b..23068ca19f7b 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -117,6 +117,8 @@ impl Backtrace { continuation: &VMContRef, f: impl FnMut(Frame) -> ControlFlow<()>, ) { + log::trace!("====== Capturing Backtrace (suspended continuation) ======"); + assert_eq!( continuation.common_stack_information.state, VMStackState::Suspended @@ -137,8 +139,16 @@ impl Backtrace { // here. let stack_chain = VMStackChain::Continuation(continuation as *const VMContRef as *mut VMContRef); - Self::trace_through_continuations(unwind, stack_chain, pc, fp, trampoline_fp, f); + + if let ControlFlow::Break(()) = + Self::trace_through_continuations(unwind, stack_chain, pc, fp, trampoline_fp, f) + { + log::trace!("====== Done Capturing Backtrace (closure break) ======"); + return; + } } + + log::trace!("====== Done Capturing Backtrace (reached end of stack chain) ======"); } /// Walk the current Wasm stack, calling `f` for each frame we walk. From b35a323a5ca3f8e43e105bd2d0e67481f03fa063 Mon Sep 17 00:00:00 2001 From: Frank Emrich Date: Mon, 21 Apr 2025 22:24:00 +0100 Subject: [PATCH 4/6] [pr2] base --- cranelift/codegen/src/ir/function.rs | 7 + cranelift/codegen/src/isa/x64/abi.rs | 7 + crates/cranelift/src/func_environ.rs | 300 +- crates/cranelift/src/lib.rs | 7 +- .../src/stack_switching/control_effect.rs | 84 + .../src/stack_switching/fatpointer.rs | 71 + .../src/stack_switching/instructions.rs | 2543 +++++++++++++++++ crates/cranelift/src/stack_switching/mod.rs | 3 + .../src/translate/code_translator.rs | 166 +- 9 files changed, 3091 insertions(+), 97 deletions(-) create mode 100644 crates/cranelift/src/stack_switching/control_effect.rs create mode 100644 crates/cranelift/src/stack_switching/fatpointer.rs create mode 100644 crates/cranelift/src/stack_switching/instructions.rs create mode 100644 crates/cranelift/src/stack_switching/mod.rs diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index 26070c1bad8d..d6dda0b97d1c 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -255,6 +255,13 @@ impl FunctionStencil { self.get_dyn_scale(dyn_ty) } + /// Find the data for the given stack slot + pub fn get_stack_slot_data(&self, stack_slot: StackSlot) -> &StackSlotData { + self.sized_stack_slots + .get(stack_slot) + .expect("undeclared stack slot: {stack_slot}") + } + /// Get a concrete `Type` from a user defined `DynamicType`. pub fn get_concrete_dynamic_ty(&self, ty: DynamicType) -> Option { self.dfg diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 7f657bc16ba3..cf27a446a3c6 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -104,6 +104,7 @@ impl ABIMachineSpec for X64ABIMachineSpec { mut args: ArgsAccumulator, ) -> CodegenResult<(u32, Option)> { let is_fastcall = call_conv == CallConv::WindowsFastcall; + let is_tail = call_conv == CallConv::Tail; let mut next_gpr = 0; let mut next_vreg = 0; @@ -182,6 +183,11 @@ impl ABIMachineSpec for X64ABIMachineSpec { // This is consistent with LLVM's behavior, and is needed for // some uses of Cranelift (e.g., the rustc backend). // + // - Otherwise, if the calling convention is Tail, we behave as in + // the previous case, even if `enable_llvm_abi_extensions` is not + // set in the flags: This is a custom calling convention defined + // by Cranelift, LLVM doesn't know about it. + // // - Otherwise, both SysV and Fastcall specify behavior (use of // vector register, a register pair, or passing by reference // depending on the case), but for simplicity, we will just panic if @@ -195,6 +201,7 @@ impl ABIMachineSpec for X64ABIMachineSpec { if param.value_type.bits() > 64 && !(param.value_type.is_vector() || param.value_type.is_float()) && !flags.enable_llvm_abi_extensions() + && !is_tail { panic!( "i128 args/return values not supported unless LLVM ABI extensions are enabled" diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 01c22c3ddc4d..191ca8a82955 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -5,7 +5,7 @@ use crate::translate::{ FuncTranslationState, GlobalVariable, Heap, HeapData, StructFieldsVec, TableData, TableSize, TargetEnvironment, }; -use crate::{BuiltinFunctionSignatures, TRAP_INTERNAL_ASSERT}; +use crate::{stack_switching, BuiltinFunctionSignatures, TRAP_INTERNAL_ASSERT}; use cranelift_codegen::cursor::FuncCursor; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; use cranelift_codegen::ir::immediates::{Imm64, Offset32}; @@ -23,9 +23,9 @@ use wasmparser::{Operator, WasmFeatures}; use wasmtime_environ::{ BuiltinFunctionIndex, DataIndex, ElemIndex, EngineOrModuleTypeIndex, FuncIndex, GlobalIndex, IndexType, Memory, MemoryIndex, Module, ModuleInternedTypeIndex, ModuleTranslation, - ModuleTypesBuilder, PtrSize, Table, TableIndex, TripleExt, Tunables, TypeConvert, TypeIndex, - VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType, WasmRefType, - WasmResult, WasmValType, + ModuleTypesBuilder, PtrSize, Table, TableIndex, TagIndex, TripleExt, Tunables, TypeConvert, + TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType, + WasmRefType, WasmResult, WasmValType, }; use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK}; @@ -92,9 +92,9 @@ wasmtime_environ::foreach_builtin_function!(declare_function_signatures); /// The `FuncEnvironment` implementation for use by the `ModuleEnvironment`. pub struct FuncEnvironment<'module_environment> { compiler: &'module_environment Compiler, - isa: &'module_environment (dyn TargetIsa + 'module_environment), - module: &'module_environment Module, - types: &'module_environment ModuleTypesBuilder, + pub(crate) isa: &'module_environment (dyn TargetIsa + 'module_environment), + pub(crate) module: &'module_environment Module, + pub(crate) types: &'module_environment ModuleTypesBuilder, wasm_func_ty: &'module_environment WasmFuncType, sig_ref_to_ty: SecondaryMap>, needs_gc_heap: bool, @@ -136,7 +136,7 @@ pub struct FuncEnvironment<'module_environment> { pcc_vmctx_memtype: Option, /// Caches of signatures for builtin functions. - builtin_functions: BuiltinFunctions, + pub(crate) builtin_functions: BuiltinFunctions, /// Offsets to struct fields accessed by JIT code. pub(crate) offsets: VMOffsets, @@ -172,6 +172,16 @@ pub struct FuncEnvironment<'module_environment> { /// always present even if this is a "leaf" function, as we have to call /// into the host to trap when signal handlers are disabled. pub(crate) stack_limit_at_function_entry: Option, + + /// Used by the stack switching feature. If set, we have a allocated a + /// slot on this function's stack to be used for the + /// current stack's `handler_list` field. + pub(crate) stack_switching_handler_list_buffer: Option, + + /// Used by the stack switching feature. If set, we have a allocated a + /// slot on this function's stack to be used for the + /// current continuation's `values` field. + pub(crate) stack_switching_values_buffer: Option, } impl<'module_environment> FuncEnvironment<'module_environment> { @@ -226,6 +236,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> { translation, stack_limit_at_function_entry: None, + + stack_switching_handler_list_buffer: None, + stack_switching_values_buffer: None, } } @@ -1509,8 +1522,6 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { return CheckIndirectCallTypeSignature::StaticTrap; } - WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. - // Engine-indexed types don't show up until runtime and it's a Wasm // validation error to perform a call through a non-function table, // so these cases are dynamically not reachable. @@ -1525,6 +1536,9 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { | WasmHeapType::ConcreteArray(_) | WasmHeapType::Struct | WasmHeapType::ConcreteStruct(_) + | WasmHeapType::Cont + | WasmHeapType::ConcreteCont(_) + | WasmHeapType::NoCont | WasmHeapType::None => { unreachable!() } @@ -1754,7 +1768,9 @@ impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environm let needs_stack_map = match wasm_ty.top() { WasmHeapTopType::Extern | WasmHeapTopType::Any => true, WasmHeapTopType::Func => false, - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + // TODO(#10248) Once continuations can be stored on the GC heap, we + // will need stack maps for continuation objects. + WasmHeapTopType::Cont => false, }; (ty, needs_stack_map) } @@ -1824,22 +1840,42 @@ impl FuncEnvironment<'_> { let mut pos = builder.cursor(); let table = self.table(table_index); let ty = table.ref_type.heap_type; + let vmctx = self.vmctx_val(&mut pos); + let index_type = table.idx_type; + let delta = self.cast_index_to_i64(&mut builder.cursor(), delta, index_type); + let table_index_arg = builder.ins().iconst(I32, table_index.as_u32() as i64); + let mut args = vec![vmctx, table_index_arg, delta]; let grow = if ty.is_vmgcref_type() { - gc::builtins::table_grow_gc_ref(self, &mut pos.func)? + args.push(init_value); + gc::builtins::table_grow_gc_ref(self, &mut builder.cursor().func)? } else { - debug_assert_eq!(ty.top(), WasmHeapTopType::Func); - self.builtin_functions.table_grow_func_ref(&mut pos.func) - }; + debug_assert!(matches!( + ty.top(), + WasmHeapTopType::Func | WasmHeapTopType::Cont + )); + match ty.top() { + WasmHeapTopType::Func => { + args.push(init_value); + self.builtin_functions + .table_grow_func_ref(&mut builder.func) + } + WasmHeapTopType::Cont => { + let (revision, contref) = stack_switching::fatpointer::deconstruct( + self, + &mut builder.cursor(), + init_value, + ); + args.extend_from_slice(&[contref, revision]); + self.builtin_functions + .table_grow_cont_obj(&mut builder.func) + } - let vmctx = self.vmctx_val(&mut pos); + _ => panic!("unsupported table type."), + } + }; - let index_type = table.idx_type; - let delta = self.cast_index_to_i64(&mut pos, delta, index_type); - let table_index_arg = pos.ins().iconst(I32, table_index.as_u32() as i64); - let call_inst = pos - .ins() - .call(grow, &[vmctx, table_index_arg, delta, init_value]); - let result = pos.func.dfg.first_result(call_inst); + let call_inst = builder.ins().call(grow, &args); + let result = builder.func.dfg.first_result(call_inst); Ok(self.convert_pointer_to_index_type(builder.cursor(), result, index_type, false)) } @@ -1872,7 +1908,16 @@ impl FuncEnvironment<'_> { } // Continuation types. - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => { + self.ensure_table_exists(builder.func, table_index); + let (table_entry_addr, flags) = table_data.prepare_table_addr(self, builder, index); + Ok(builder.ins().load( + stack_switching::fatpointer::POINTER_TYPE, + flags, + table_entry_addr, + 0, + )) + } } } @@ -1921,7 +1966,11 @@ impl FuncEnvironment<'_> { } // Continuation types. - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => { + let (elem_addr, flags) = table_data.prepare_table_addr(self, builder, index); + builder.ins().store(flags, value, elem_addr, 0); + Ok(()) + } } } @@ -1935,22 +1984,36 @@ impl FuncEnvironment<'_> { ) -> WasmResult<()> { let mut pos = builder.cursor(); let table = self.table(table_index); - let index_type = table.idx_type; - let dst = self.cast_index_to_i64(&mut pos, dst, index_type); - let len = self.cast_index_to_i64(&mut pos, len, index_type); let ty = table.ref_type.heap_type; + let vmctx = self.vmctx_val(&mut pos); + let index_type = table.idx_type; + let table_index_arg = builder.ins().iconst(I32, table_index.as_u32() as i64); + let dst = self.cast_index_to_i64(&mut builder.cursor(), dst, index_type); + let len = self.cast_index_to_i64(&mut builder.cursor(), len, index_type); + let mut args = vec![vmctx, table_index_arg, dst]; let libcall = if ty.is_vmgcref_type() { - gc::builtins::table_fill_gc_ref(self, &mut pos.func)? + args.push(val); + gc::builtins::table_fill_gc_ref(self, &mut builder.cursor().func)? } else { - debug_assert_eq!(ty.top(), WasmHeapTopType::Func); - self.builtin_functions.table_fill_func_ref(&mut pos.func) + match ty.top() { + WasmHeapTopType::Func => { + args.push(val); + self.builtin_functions + .table_fill_func_ref(&mut builder.func) + } + WasmHeapTopType::Cont => { + let (revision, contref) = + stack_switching::fatpointer::deconstruct(self, &mut builder.cursor(), val); + args.extend_from_slice(&[contref, revision]); + self.builtin_functions + .table_fill_cont_obj(&mut builder.func) + } + _ => panic!("unsupported table type"), + } }; + args.push(len); - let vmctx = self.vmctx_val(&mut pos); - - let table_index_arg = pos.ins().iconst(I32, table_index.as_u32() as i64); - pos.ins() - .call(libcall, &[vmctx, table_index_arg, dst, val, len]); + builder.ins().call(libcall, &args); Ok(()) } @@ -2271,7 +2334,10 @@ impl FuncEnvironment<'_> { WasmHeapTopType::Func => pos.ins().iconst(self.pointer_type(), 0), // NB: null GC references don't need to be in stack maps. WasmHeapTopType::Any | WasmHeapTopType::Extern => pos.ins().iconst(types::I32, 0), - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => { + let zero = pos.ins().iconst(self.pointer_type(), 0); + stack_switching::fatpointer::construct(self, &mut pos, zero, zero) + } }) } @@ -2280,9 +2346,19 @@ impl FuncEnvironment<'_> { mut pos: cranelift_codegen::cursor::FuncCursor, value: ir::Value, ) -> WasmResult { - let byte_is_null = - pos.ins() - .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0); + let byte_is_null = match pos.func.dfg.value_type(value) { + // continuation + ty if ty == stack_switching::fatpointer::POINTER_TYPE => { + let (_revision, contref) = + stack_switching::fatpointer::deconstruct(self, &mut pos, value); + pos.ins() + .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, contref, 0) + } + _ => pos + .ins() + .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0), + }; + Ok(pos.ins().uextend(ir::types::I32, byte_is_null)) } @@ -3215,6 +3291,133 @@ impl FuncEnvironment<'_> { self.isa.triple().architecture == target_lexicon::Architecture::X86_64 } + pub fn translate_cont_bind( + &mut self, + builder: &mut FunctionBuilder<'_>, + contobj: ir::Value, + args: &[ir::Value], + ) -> ir::Value { + stack_switching::instructions::translate_cont_bind(self, builder, contobj, args) + } + + pub fn translate_cont_new( + &mut self, + builder: &mut FunctionBuilder<'_>, + _state: &FuncTranslationState, + func: ir::Value, + arg_types: &[WasmValType], + return_types: &[WasmValType], + ) -> WasmResult { + stack_switching::instructions::translate_cont_new( + self, + builder, + func, + arg_types, + return_types, + ) + } + + pub fn translate_resume( + &mut self, + builder: &mut FunctionBuilder<'_>, + type_index: u32, + contobj: ir::Value, + resume_args: &[ir::Value], + resumetable: &[(u32, Option)], + ) -> WasmResult> { + stack_switching::instructions::translate_resume( + self, + builder, + type_index, + contobj, + resume_args, + resumetable, + ) + } + + #[allow(dead_code, reason = "TODO")] + pub fn translate_resume_throw( + &mut self, + _pos: FuncCursor, + _state: &FuncTranslationState, + _tag_index: u32, + _cont: ir::Value, + ) -> WasmResult { + // TODO(#10248) + Err(wasmtime_environ::WasmError::Unsupported( + "resume.throw instruction not implemented, yet".to_string(), + )) + } + + pub fn translate_suspend( + &mut self, + builder: &mut FunctionBuilder<'_>, + tag_index: u32, + suspend_args: &[ir::Value], + tag_return_types: &[ir::Type], + ) -> Vec { + stack_switching::instructions::translate_suspend( + self, + builder, + tag_index, + suspend_args, + tag_return_types, + ) + } + + /// Translates switch instructions. + pub fn translate_switch( + &mut self, + builder: &mut FunctionBuilder, + tag_index: u32, + contobj: ir::Value, + switch_args: &[ir::Value], + return_types: &[ir::Type], + ) -> WasmResult> { + stack_switching::instructions::translate_switch( + self, + builder, + tag_index, + contobj, + switch_args, + return_types, + ) + } + + pub fn continuation_arguments(&self, index: TypeIndex) -> &[WasmValType] { + let idx = self.module.types[index].unwrap_module_type_index(); + self.types[self.types[idx] + .unwrap_cont() + .clone() + .unwrap_interned_type_index()] + .unwrap_func() + .params() + } + + pub fn continuation_returns(&self, index: TypeIndex) -> &[WasmValType] { + let idx = self.module.types[index].unwrap_module_type_index(); + self.types[self.types[idx] + .unwrap_cont() + .clone() + .unwrap_interned_type_index()] + .unwrap_func() + .returns() + } + + pub fn tag_params(&self, tag_index: TagIndex) -> &[WasmValType] { + let idx = self.module.tags[tag_index].signature; + self.types[idx.unwrap_module_type_index()] + .unwrap_func() + .params() + } + + pub fn tag_returns(&self, tag_index: TagIndex) -> &[WasmValType] { + let idx = self.module.tags[tag_index].signature; + self.types[idx.unwrap_module_type_index()] + .unwrap_func() + .returns() + } + pub fn use_x86_blendv_for_relaxed_laneselect(&self, ty: Type) -> bool { self.isa.has_x86_blendv_lowering(ty) } @@ -3515,20 +3718,3 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type { IndexType::I64 => I64, } } - -/// TODO(10248) This is removed in the next stack switching PR. It stops the -/// compiler from complaining about the stack switching libcalls being dead -/// code. -#[allow( - dead_code, - reason = "Dummy function to supress more dead code warnings" -)] -pub fn use_stack_switching_libcalls() { - let _ = BuiltinFunctions::delete_me_print_str; - let _ = BuiltinFunctions::delete_me_print_int; - let _ = BuiltinFunctions::delete_me_print_pointer; - - let _ = BuiltinFunctions::cont_new; - let _ = BuiltinFunctions::table_grow_cont_obj; - let _ = BuiltinFunctions::table_fill_cont_obj; -} diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index 60ab8ebe72bd..194f202ec0c7 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -35,6 +35,7 @@ mod builder; mod compiler; mod debug; mod func_environ; +mod stack_switching; mod translate; use self::compiler::Compiler; @@ -208,11 +209,7 @@ fn reference_type(wasm_ht: WasmHeapType, pointer_type: ir::Type) -> ir::Type { match wasm_ht.top() { WasmHeapTopType::Func => pointer_type, WasmHeapTopType::Any | WasmHeapTopType::Extern => ir::types::I32, - WasmHeapTopType::Cont => - // TODO(10248) This is added in a follow-up PR - { - unimplemented!("codegen for stack switching types not implemented, yet") - } + WasmHeapTopType::Cont => stack_switching::fatpointer::POINTER_TYPE, } } diff --git a/crates/cranelift/src/stack_switching/control_effect.rs b/crates/cranelift/src/stack_switching/control_effect.rs new file mode 100644 index 000000000000..1894b1a3a421 --- /dev/null +++ b/crates/cranelift/src/stack_switching/control_effect.rs @@ -0,0 +1,84 @@ +use cranelift_codegen::ir; +use cranelift_codegen::ir::types::{I32, I64}; +use cranelift_codegen::ir::InstBuilder; +use cranelift_frontend::FunctionBuilder; +use wasmtime_environ::stack_switching as stack_switching_environ; + +/// Universal control effect. This structure encodes return signal, +/// resume signal, suspension signal, and handler index into a +/// u64 value. This instance is used at compile time. There is a runtime +/// counterpart in `continuations/src/lib.rs`. +/// We convert to and from u64 as follows: The low 32 bits of the u64 are the +/// discriminant, the high 32 bits are the handler_index (if `Suspend`) +#[derive(Clone, Copy)] +pub struct ControlEffect(ir::Value); + +impl ControlEffect { + // Returns the discriminant + pub fn signal<'a>( + &self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + builder.ins().ushr_imm(self.0, 32) + } + + pub fn from_u64(val: ir::Value) -> Self { + Self(val) + } + + pub fn to_u64(&self) -> ir::Value { + self.0 + } + + pub fn encode_resume<'a>( + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> Self { + let discriminant = builder.ins().iconst( + I64, + i64::from(stack_switching_environ::CONTROL_EFFECT_RESUME_DISCRIMINANT), + ); + let val = builder.ins().ishl_imm(discriminant, 32); + + Self(val) + } + + pub fn encode_switch<'a>( + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> Self { + let discriminant = builder.ins().iconst( + I64, + i64::from(stack_switching_environ::CONTROL_EFFECT_SWITCH_DISCRIMINANT), + ); + let val = builder.ins().ishl_imm(discriminant, 32); + + Self(val) + } + + pub fn encode_suspend<'a>( + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + handler_index: ir::Value, + ) -> Self { + let discriminant = builder.ins().iconst( + I64, + i64::from(stack_switching_environ::CONTROL_EFFECT_SUSPEND_DISCRIMINANT), + ); + let val = builder.ins().ishl_imm(discriminant, 32); + let handler_index = builder.ins().uextend(I64, handler_index); + let val = builder.ins().bor(val, handler_index); + + Self(val) + } + + /// Returns the payload of the `Suspend` variant + pub fn handler_index<'a>( + self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + builder.ins().ireduce(I32, self.0) + } +} diff --git a/crates/cranelift/src/stack_switching/fatpointer.rs b/crates/cranelift/src/stack_switching/fatpointer.rs new file mode 100644 index 000000000000..d62f4b04576e --- /dev/null +++ b/crates/cranelift/src/stack_switching/fatpointer.rs @@ -0,0 +1,71 @@ +use cranelift_codegen::ir; +use cranelift_codegen::ir::types::I64; +use cranelift_codegen::ir::InstBuilder; + +/// The Cranelfift type used to represent all of the following: +/// - wasm values of type `(ref null $ct)` and `(ref $ct)` +/// - equivalently: runtime values of type `Option` and `VMContObj` +/// Note that a `VMContObj` is a fat pointer +/// consisting of a pointer to `VMContRef` and a 64 bit sequence +/// counter. +/// We represent this here as a 128bit value, with the same representation as +/// `core::mem::transmute::`. +pub const POINTER_TYPE: ir::Type = ir::types::I128; + +/// Turns a (possibly null) reference to a continuation object into a tuple +/// (revision, contref_ptr). If `contobj` denotes a wasm null reference, the +/// contref_ptr part will be a null pointer. +pub(crate) fn deconstruct<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + pos: &mut cranelift_codegen::cursor::FuncCursor, + contobj: ir::Value, +) -> (ir::Value, ir::Value) { + debug_assert_eq!(pos.func.dfg.value_type(contobj), POINTER_TYPE); + + let (lsbs, msbs) = pos.ins().isplit(contobj); + + let (revision_counter, contref) = match env.isa.endianness() { + ir::Endianness::Little => (lsbs, msbs), + ir::Endianness::Big => { + let pad_bits = 64 - env.pointer_type().bits(); + let contref = pos.ins().ushr_imm(lsbs, pad_bits as i64); + (msbs, contref) + } + }; + let contref = if env.pointer_type().bits() < I64.bits() { + pos.ins().ireduce(env.pointer_type(), contref) + } else { + contref + }; + (revision_counter, contref) +} + +/// Constructs a continuation object from a given contref and revision pointer. +/// The contref_addr may be 0, to indicate that we want to build a wasm null reference. +pub(crate) fn construct<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + pos: &mut cranelift_codegen::cursor::FuncCursor, + revision_counter: ir::Value, + contref_addr: ir::Value, +) -> ir::Value { + debug_assert_eq!(pos.func.dfg.value_type(contref_addr), env.pointer_type()); + debug_assert_eq!(pos.func.dfg.value_type(revision_counter), ir::types::I64); + let contref_addr = if env.pointer_type().bits() < I64.bits() { + pos.ins().uextend(I64, contref_addr) + } else { + contref_addr + }; + let (msbs, lsbs) = match env.isa.endianness() { + ir::Endianness::Little => (contref_addr, revision_counter), + ir::Endianness::Big => { + let pad_bits = 64 - env.pointer_type().bits(); + let lsbs = pos.ins().ishl_imm(contref_addr, pad_bits as i64); + (revision_counter, lsbs) + } + }; + + let lsbs = pos.ins().uextend(ir::types::I128, lsbs); + let msbs = pos.ins().uextend(ir::types::I128, msbs); + let msbs = pos.ins().ishl_imm(msbs, 64); + pos.ins().bor(lsbs, msbs) +} diff --git a/crates/cranelift/src/stack_switching/instructions.rs b/crates/cranelift/src/stack_switching/instructions.rs new file mode 100644 index 000000000000..8352045f2434 --- /dev/null +++ b/crates/cranelift/src/stack_switching/instructions.rs @@ -0,0 +1,2543 @@ +use cranelift_codegen::ir::BlockArg; +use itertools::{Either, Itertools}; + +use cranelift_codegen::ir::condcodes::*; +use cranelift_codegen::ir::types::*; +use cranelift_codegen::ir::{self, MemFlags}; +use cranelift_codegen::ir::{Block, BlockCall, InstBuilder, JumpTableData}; +use cranelift_frontend::FunctionBuilder; +use wasmtime_environ::stack_switching as stack_switching_environ; +use wasmtime_environ::{PtrSize, TagIndex, TypeIndex, WasmResult, WasmValType}; + +pub const DEBUG_ASSERT_TRAP_CODE: crate::TrapCode = crate::TRAP_DELETE_ME_DEBUG_ASSERTION; + +// TODO(frank-emrich) This is the size for x64 Linux. Once we support different +// platforms for stack switching, must select appropriate value for target. +pub const CONTROL_CONTEXT_SIZE: usize = 24; + +use super::control_effect::ControlEffect; +use super::fatpointer; + +// FIXME(frank-emrich) The debugging facilities in this module are very unsafe +// (see comment on `emit_debug_print`). They are not supposed to be part of the +// final, upstreamed code, but deleted beforehand. +#[macro_use] +pub(crate) mod delete_me { + use cranelift_codegen::ir; + use cranelift_codegen::ir::condcodes::IntCC; + use cranelift_codegen::ir::types::*; + use cranelift_codegen::ir::InstBuilder; + use cranelift_frontend::FunctionBuilder; + + macro_rules! call_builtin { + ( $builder:ident, $env:ident, $f:ident( $($args:expr),* ) ) => ( + { + let fname = $env.builtin_functions.$f(&mut $builder.func); + let vmctx = $env.vmctx_val(&mut $builder.cursor()); + $builder.ins().call(fname, &[vmctx, $( $args ), * ]); + } + ); + } + + /// FIXME(frank-emrich) This printing functionality is inherently unsafe: It + /// hard-codes the addresses of the string literals it uses, without any + /// relocation information. Therefore, it will immediately crash and burn if + /// the compiled code is ever used in a different execution of wasmtime than + /// the one producing it. + /// As a result it is not supposed to be part of the final, upstreamed code. + /// + /// Low-level implementation of debug printing. Do not use directly; see + /// `emit_debug_println!` macro for doing actual printing. + /// + /// Takes a string literal which may contain placeholders similarly to those + /// supported by `std::fmt`. + /// + /// Currently supported placeholders: + /// {} for unsigned integers + /// {:p} for printing pointers (in hex form) + /// + /// When printing, we replace them with the corresponding values in `vals`. + /// Thus, the number of placeholders in `s` must match the number of entries + /// in `vals`. + pub fn emit_debug_print<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + s: &'static str, + vals: &[ir::Value], + ) { + let print_s_infix = |env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + start: usize, + end: usize| { + if start < end { + let s: &'static str = &s[start..end]; + // This is quite dodgy, which is why we can only do this for + // debugging purposes: + // At jit time, we take a pointer to the slice of the (static) + // string, thus yielding an address within wasmtime's DATA + // section. This pointer is hard-code into generated code. We do + // not emit any kind of relocation information, which means that + // this breaks if we were to store the generated code and use it + // during subsequent executions of wasmtime (e.g., when using + // wasmtime compile). + let ptr = s.as_ptr(); + let ptr = builder.ins().iconst(env.pointer_type(), ptr as i64); + let len = s.len(); + let len = builder.ins().iconst(I64, len as i64); + + call_builtin!(builder, env, delete_me_print_str(ptr, len)); + } + }; + let print_int = |env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + val: ir::Value| { + let ty = builder.func.dfg.value_type(val); + let val = match ty { + I8 | I32 => builder.ins().uextend(I64, val), + I64 => val, + _ => panic!("Cannot print type {ty}"), + }; + call_builtin!(builder, env, delete_me_print_int(val)); + }; + let print_pointer = |env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ptr: ir::Value| { + call_builtin!(builder, env, delete_me_print_pointer(ptr)); + }; + + if super::stack_switching_environ::ENABLE_DEBUG_PRINTING { + let mut prev_end = 0; + let mut i = 0; + + let mut ph_matches: Vec<(usize, &'static str)> = s + .match_indices("{}") + .chain(s.match_indices("{:p}")) + .collect(); + ph_matches.sort_by_key(|(index, _)| *index); + + for (start, matched_ph) in ph_matches { + let end = start + matched_ph.len(); + + assert!( + i < vals.len(), + "Must supply as many entries in vals as there are placeholders in the string" + ); + + print_s_infix(env, builder, prev_end, start); + match matched_ph { + "{}" => print_int(env, builder, vals[i]), + "{:p}" => print_pointer(env, builder, vals[i]), + u => panic!("Unsupported placeholder in debug_print input string: {u}"), + } + prev_end = end; + i += 1; + } + assert_eq!( + i, + vals.len(), + "Must supply as many entries in vals as there are placeholders in the string" + ); + + print_s_infix(env, builder, prev_end, s.len()); + } + } + + /// Emits code to print debug information. Only actually prints in debug + /// builds and if debug printing flag is enabled. The third and all + /// following arguments are like those to println!: A string literal with + /// placeholders followed by the actual values. + /// + /// Summary of arguments: + /// * `env` - Type &mut crate::func_environ::FuncEnvironment<'a> + /// * `builder` - Type &mut FunctionBuilder, + /// * `msg` : String literal, containing placeholders like those supported by println! + /// * remaining arguments: ir::Values filled into the placeholders in `msg` + #[allow(unused_macros, reason = "Only used in certain debug builds")] + macro_rules! emit_debug_println { + ($env : expr, $builder : expr, $msg : literal, $( $arg:expr ),*) => { + let msg_newline : &'static str= std::concat!( + $msg, + "\n" + ); + emit_debug_print($env, $builder, msg_newline, &[$($arg),*]); + } + } + + /// Low-level implementation of assertion mechanism. Use emit_debug_* macros + /// instead. + /// + /// If `ENABLE_DEBUG_PRINTING` is enabled, `error_str` is printed before + /// trapping in case of an assertion violation. + pub fn emit_debug_assert_generic<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + condition: ir::Value, + error_str: &'static str, + ) { + if cfg!(debug_assertions) { + if super::stack_switching_environ::ENABLE_DEBUG_PRINTING { + let failure_block = builder.create_block(); + let continue_block = builder.create_block(); + + builder + .ins() + .brif(condition, continue_block, &[], failure_block, &[]); + + builder.switch_to_block(failure_block); + builder.seal_block(failure_block); + + emit_debug_print(env, builder, error_str, &[]); + builder.ins().debugtrap(); + builder.ins().jump(continue_block, &[]); + + builder.switch_to_block(continue_block); + builder.seal_block(continue_block); + } else { + builder + .ins() + .trapz(condition, super::DEBUG_ASSERT_TRAP_CODE); + } + } + } + + /// Low-level implementation of assertion mechanism. Use emit_debug_* macros + /// instead. + /// + /// If `ENABLE_DEBUG_PRINTING` is enabled, `error_str` is printed before + /// trapping in case of an assertion violation. Here, `error_str` is expected + /// to contain two placeholders, such as {} or {:p}, which are replaced with + /// `v1` and `v2` when printing. + pub fn emit_debug_assert_icmp<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + operator: IntCC, + v1: ir::Value, + v2: ir::Value, + error_str: &'static str, + ) { + if cfg!(debug_assertions) { + let cmp_res = builder.ins().icmp(operator, v1, v2); + + if super::stack_switching_environ::ENABLE_DEBUG_PRINTING { + let failure_block = builder.create_block(); + let continue_block = builder.create_block(); + + builder + .ins() + .brif(cmp_res, continue_block, &[], failure_block, &[]); + + builder.switch_to_block(failure_block); + builder.seal_block(failure_block); + + emit_debug_print(env, builder, error_str, &[v1, v2]); + builder.ins().debugtrap(); + builder.ins().jump(continue_block, &[]); + + builder.switch_to_block(continue_block); + builder.seal_block(continue_block); + } else { + builder.ins().trapz(cmp_res, super::DEBUG_ASSERT_TRAP_CODE); + } + } + } + + /// Used to implement other macros, do not use directly. + macro_rules! emit_debug_assert_icmp { + ( $env : expr, + $builder: expr, + $operator : expr, + $operator_string : expr, + $v1 : expr, + $v2 : expr) => { + let msg: &'static str = std::concat!( + "assertion failure in ", + std::file!(), + ", line ", + std::line!(), + ": {} ", + $operator_string, + " {} does not hold\n" + ); + emit_debug_assert_icmp($env, $builder, $operator, $v1, $v2, msg); + }; + } + + macro_rules! emit_debug_assert { + ($env: expr, $builder: expr, $condition: expr) => { + let msg: &'static str = std::concat!( + "assertion failure in ", + std::file!(), + ", line ", + std::line!(), + "\n" + ); + // This makes the borrow checker happy if $condition uses env or builder. + let c = $condition; + emit_debug_assert_generic($env, $builder, c, msg); + }; + } + + macro_rules! emit_debug_assert_eq { + ($env: expr, $builder: expr, $v1 : expr, $v2: expr) => { + emit_debug_assert_icmp!($env, $builder, IntCC::Equal, "==", $v1, $v2); + }; + } + + macro_rules! emit_debug_assert_ne { + ($env: expr, $builder: expr, $v1 : expr, $v2: expr) => { + emit_debug_assert_icmp!($env, $builder, IntCC::NotEqual, "!=", $v1, $v2); + }; + } + + macro_rules! emit_debug_assert_ule { + ($env: expr, $builder: expr, $v1 : expr, $v2: expr) => { + emit_debug_assert_icmp!( + $env, + $builder, + IntCC::UnsignedLessThanOrEqual, + "<=", + $v1, + $v2 + ); + }; + } +} +use delete_me::*; + +/// This module contains compile-time counterparts to types defined elsewhere. +pub(crate) mod stack_switching_helpers { + use super::delete_me::*; + use core::marker::PhantomData; + use cranelift_codegen::ir; + use cranelift_codegen::ir::condcodes::IntCC; + use cranelift_codegen::ir::types::*; + use cranelift_codegen::ir::InstBuilder; + use cranelift_codegen::ir::{StackSlot, StackSlotKind::*}; + use cranelift_frontend::FunctionBuilder; + use std::mem; + use wasmtime_environ::PtrSize; + + #[derive(Copy, Clone)] + pub struct VMContRef { + pub address: ir::Value, + } + + #[derive(Copy, Clone)] + pub struct VMArray { + /// Base address of this object, which must be shifted by `offset` below. + base: ir::Value, + + /// Adding this (statically) known offset gets us the overall address. + offset: i32, + + /// The type parameter T is never used in the fields above. We still + /// want to have it for consistency with + /// `stack_switching_environ::Vector` and to use it in the associated + /// functions. + phantom: PhantomData, + } + + pub type VMPayloads = VMArray; + + // Actually a vector of *mut VMTagDefinition + pub type VMHandlerList = VMArray<*mut u8>; + + /// Compile-time representation of stack_switching_environ::VMStackChain, + /// consisting of two `ir::Value`s. + pub struct VMStackChain { + discriminant: ir::Value, + payload: ir::Value, + } + + pub struct VMCommonStackInformation { + pub address: ir::Value, + } + + /// Compile-time representation of `crate::runtime::vm::stack::VMContinuationStack`. + pub struct VMContinuationStack { + /// This is NOT the "top of stack" address of the stack itself. In line + /// with how the (runtime) `FiberStack` type works, this is a pointer to + /// the TOS address. + tos_ptr: ir::Value, + } + + impl VMContRef { + pub fn new(address: ir::Value) -> VMContRef { + VMContRef { address } + } + + pub fn args<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + _builder: &mut FunctionBuilder, + ) -> VMPayloads { + let offset = env.offsets.ptr.vmcontref_args() as i32; + VMPayloads::new(self.address, offset) + } + + pub fn values<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + _builder: &mut FunctionBuilder, + ) -> VMPayloads { + let offset = env.offsets.ptr.vmcontref_values() as i32; + VMPayloads::new(self.address, offset) + } + + pub fn common_stack_information<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMCommonStackInformation { + let offset = env.offsets.ptr.vmcontref_common_stack_information() as i64; + let address = builder.ins().iadd_imm(self.address, offset); + VMCommonStackInformation { address } + } + + /// Stores the parent of this continuation, which may either be another + /// continuation or the initial stack. It is therefore represented as a + /// `VMStackChain` element. + pub fn set_parent_stack_chain<'a>( + &mut self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + new_stack_chain: &VMStackChain, + ) { + let offset = env.offsets.ptr.vmcontref_parent_chain() as i32; + new_stack_chain.store(env, builder, self.address, offset) + } + + /// Loads the parent of this continuation, which may either be another + /// continuation or the initial stack. It is therefore represented as a + /// `VMStackChain` element. + pub fn get_parent_stack_chain<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMStackChain { + let offset = env.offsets.ptr.vmcontref_parent_chain() as i32; + VMStackChain::load(env, builder, self.address, offset, env.pointer_type()) + } + + pub fn set_last_ancestor<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + last_ancestor: ir::Value, + ) { + let offset = env.offsets.ptr.vmcontref_last_ancestor() as i32; + let mem_flags = ir::MemFlags::trusted(); + builder + .ins() + .store(mem_flags, last_ancestor, self.address, offset); + } + + pub fn get_last_ancestor<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let offset = env.offsets.ptr.vmcontref_last_ancestor() as i32; + let mem_flags = ir::MemFlags::trusted(); + builder + .ins() + .load(env.pointer_type(), mem_flags, self.address, offset) + } + + /// Gets the revision counter the a given continuation + /// reference. + pub fn get_revision<'a>( + &mut self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let mem_flags = ir::MemFlags::trusted(); + let offset = env.offsets.ptr.vmcontref_revision() as i32; + let revision = builder.ins().load(I64, mem_flags, self.address, offset); + revision + } + + /// Sets the revision counter on the given continuation + /// reference to `revision + 1`. + + pub fn incr_revision<'a>( + &mut self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + revision: ir::Value, + ) -> ir::Value { + if cfg!(debug_assertions) { + let actual_revision = self.get_revision(env, builder); + emit_debug_assert_eq!(env, builder, revision, actual_revision); + } + let mem_flags = ir::MemFlags::trusted(); + let offset = env.offsets.ptr.vmcontref_revision() as i32; + let revision_plus1 = builder.ins().iadd_imm(revision, 1); + builder + .ins() + .store(mem_flags, revision_plus1, self.address, offset); + if cfg!(debug_assertions) { + let new_revision = self.get_revision(env, builder); + emit_debug_assert_eq!(env, builder, revision_plus1, new_revision); + // Check for overflow: + emit_debug_assert_ule!(env, builder, revision, revision_plus1); + } + revision_plus1 + } + + pub fn get_fiber_stack<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMContinuationStack { + // The top of stack field is stored at offset 0 of the `FiberStack`. + let offset = env.offsets.ptr.vmcontref_stack() as i64; + let fiber_stack_top_of_stack_ptr = builder.ins().iadd_imm(self.address, offset); + VMContinuationStack::new(fiber_stack_top_of_stack_ptr) + } + } + + impl VMArray { + pub(crate) fn new(base: ir::Value, offset: i32) -> Self { + Self { + base, + offset, + phantom: PhantomData::default(), + } + } + + fn get(&self, builder: &mut FunctionBuilder, ty: ir::Type, offset: i32) -> ir::Value { + let mem_flags = ir::MemFlags::trusted(); + builder + .ins() + .load(ty, mem_flags, self.base, self.offset + offset) + } + + fn set(&self, builder: &mut FunctionBuilder, offset: i32, value: ir::Value) { + debug_assert_eq!( + builder.func.dfg.value_type(value), + Type::int_with_byte_size(u16::try_from(std::mem::size_of::()).unwrap()).unwrap() + ); + let mem_flags = ir::MemFlags::trusted(); + builder + .ins() + .store(mem_flags, value, self.base, self.offset + offset); + } + + pub fn get_data<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let offset = env.offsets.ptr.vmarray_data() as i32; + self.get(builder, env.pointer_type(), offset) + } + + fn get_capacity<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + // Array capacity is stored as u32. + let offset = env.offsets.ptr.vmarray_capacity() as i32; + self.get(builder, I32, offset) + } + + pub fn get_length<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + // Array length is stored as u32. + let offset = env.offsets.ptr.vmarray_length() as i32; + self.get(builder, I32, offset) + } + + fn set_length<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + length: ir::Value, + ) { + // Array length is stored as u32. + let offset = env.offsets.ptr.vmarray_length() as i32; + self.set::(builder, offset, length); + } + + fn set_capacity<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + capacity: ir::Value, + ) { + // Array capacity is stored as u32. + let offset = env.offsets.ptr.vmarray_capacity() as i32; + self.set::(builder, offset, capacity); + } + + fn set_data<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + data: ir::Value, + ) { + let offset = env.offsets.ptr.vmarray_data() as i32; + self.set::<*mut T>(builder, offset, data); + } + + /// Returns pointer to next empty slot in data buffer and marks the + /// subsequent `arg_count` slots as occupied. + pub fn occupy_next_slots<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + arg_count: i32, + ) -> ir::Value { + let data = self.get_data(env, builder); + let original_length = self.get_length(env, builder); + let new_length = builder.ins().iadd_imm(original_length, arg_count as i64); + self.set_length(env, builder, new_length); + + if cfg!(debug_assertions) { + let capacity = self.get_capacity(env, builder); + emit_debug_assert_ule!(env, builder, new_length, capacity); + } + + let value_size = mem::size_of::() as i64; + let original_length = builder.ins().uextend(I64, original_length); + let byte_offset = builder.ins().imul_imm(original_length, value_size); + builder.ins().iadd(data, byte_offset) + } + + pub fn allocate_or_reuse_stack_slot<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + required_capacity: u32, + existing_slot: Option, + ) -> StackSlot { + let zero = builder.ins().iconst(ir::types::I32, 0); + if cfg!(debug_assertions) { + // We must only allocate while there is no data in the buffer. + let length = self.get_length(env, builder); + emit_debug_assert_eq!(env, builder, length, zero); + let capacity = self.get_capacity(env, builder); + emit_debug_assert_eq!(env, builder, capacity, zero); + } + + let align = u8::try_from(std::mem::align_of::()).unwrap(); + let entry_size = u32::try_from(std::mem::size_of::()).unwrap(); + let required_size = required_capacity * entry_size; + + match existing_slot { + Some(slot) if builder.func.get_stack_slot_data(slot).size >= required_size => { + let slot_data = builder.func.get_stack_slot_data(slot).clone(); + let existing_capacity = slot_data.size / entry_size; + + let capacity_value = builder.ins().iconst(I32, existing_capacity as i64); + emit_debug_println!( + env, + builder, + "[Array::allocate_or_reuse_stack_slot] Reusing existing buffer with capacity {}", + capacity_value + ); + debug_assert!(align <= builder.func.get_stack_slot_data(slot).align_shift); + debug_assert_eq!(builder.func.get_stack_slot_data(slot).kind, ExplicitSlot); + + let existing_data = builder.ins().stack_addr(env.pointer_type(), slot, 0); + + self.set_capacity(env, builder, capacity_value); + self.set_data(env, builder, existing_data); + + slot + } + _ => { + let capacity_value = builder.ins().iconst(I32, required_capacity as i64); + emit_debug_assert_ne!(env, builder, capacity_value, zero); + + emit_debug_println!( + env, + builder, + "[Array::allocate_or_reuse_stack_slot] allocating stack slot with capacity {}", + capacity_value + ); + + let slot_size = ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + required_size, + align, + ); + let slot = builder.create_sized_stack_slot(slot_size); + let new_data = builder.ins().stack_addr(env.pointer_type(), slot, 0); + + self.set_capacity(env, builder, capacity_value); + self.set_data(env, builder, new_data); + + slot + } + } + } + + /// Loads n entries from this Vector object, where n is the length of + /// `load_types`, which also gives the types of the values to load. + /// Loading starts at index 0 of the Vector object. + pub fn load_data_entries<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + load_types: &[ir::Type], + ) -> Vec { + if cfg!(debug_assertions) { + let length = self.get_length(env, builder); + let load_count = builder.ins().iconst(I32, load_types.len() as i64); + emit_debug_assert_ule!(env, builder, load_count, length); + } + + let memflags = ir::MemFlags::trusted(); + + let data_start_pointer = self.get_data(env, builder); + let mut values = vec![]; + let mut offset = 0; + let entry_size = i32::try_from(std::mem::size_of::()).unwrap(); + for valtype in load_types { + let val = builder + .ins() + .load(*valtype, memflags, data_start_pointer, offset); + values.push(val); + offset += entry_size; + } + values + } + + /// Stores the given `values` in this Vector object, beginning at + /// index 0. This expects the Vector object to be empty (i.e., current + /// length is 0), and to be of sufficient capacity to store |`values`| + /// entries. + /// If `allow_smaller` is true, we allow storing values whose type has a + /// smaller size than T's. In that case, such values will be stored at + /// the beginning of a `T`-sized slot. + pub fn store_data_entries<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + values: &[ir::Value], + allow_smaller: bool, + ) { + let store_count = builder.ins().iconst(I32, values.len() as i64); + + if cfg!(debug_assertions) { + for val in values { + let ty = builder.func.dfg.value_type(*val); + if allow_smaller { + debug_assert!(ty.bytes() as usize <= std::mem::size_of::()); + } else { + debug_assert!(ty.bytes() as usize == std::mem::size_of::()); + } + } + + let capacity = self.get_capacity(env, builder); + let length = self.get_length(env, builder); + let zero = builder.ins().iconst(I32, 0); + emit_debug_assert_ule!(env, builder, store_count, capacity); + emit_debug_assert_eq!(env, builder, length, zero); + } + + let memflags = ir::MemFlags::trusted(); + + let data_start_pointer = self.get_data(env, builder); + + let entry_size = i32::try_from(std::mem::size_of::()).unwrap(); + let mut offset = 0; + for value in values { + builder + .ins() + .store(memflags, *value, data_start_pointer, offset); + offset += entry_size; + } + + self.set_length(env, builder, store_count); + } + + pub fn clear<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + discard_buffer: bool, + ) { + let zero32 = builder.ins().iconst(I32, 0); + self.set_length(env, builder, zero32); + + if discard_buffer { + let zero32 = builder.ins().iconst(I32, 0); + self.set_capacity(env, builder, zero32); + + let zero_ptr = builder.ins().iconst(env.pointer_type(), 0); + self.set_data(env, builder, zero_ptr); + } + } + } + + impl VMStackChain { + /// Creates a `Self` corressponding to `VMStackChain::Continuation(contref)`. + pub fn from_continuation<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + contref: ir::Value, + ) -> VMStackChain { + debug_assert_eq!( + env.offsets.ptr.size_of_vmstack_chain(), + 2 * env.offsets.ptr.size() + ); + let discriminant = + super::stack_switching_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT; + let discriminant = builder + .ins() + .iconst(env.pointer_type(), discriminant as i64); + VMStackChain { + discriminant, + payload: contref, + } + } + + /// Creates a `Self` corressponding to `VMStackChain::Absent`. + pub fn absent<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMStackChain { + debug_assert_eq!( + env.offsets.ptr.size_of_vmstack_chain(), + 2 * env.offsets.ptr.size() + ); + let discriminant = super::stack_switching_environ::STACK_CHAIN_ABSENT_DISCRIMINANT; + let discriminant = builder + .ins() + .iconst(env.pointer_type(), discriminant as i64); + let zero_filler = builder.ins().iconst(env.pointer_type(), 0i64); + VMStackChain { + discriminant, + payload: zero_filler, + } + } + + /// For debugging purposes. Emits an assertion that `self` does not correspond to + /// `VMStackChain::Absent`. + pub fn assert_not_absent<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = super::stack_switching_environ::STACK_CHAIN_ABSENT_DISCRIMINANT; + let discriminant = builder + .ins() + .iconst(env.pointer_type(), discriminant as i64); + emit_debug_assert_ne!(env, builder, self.discriminant, discriminant); + } + + pub fn is_initial_stack<'a>( + &self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + builder.ins().icmp_imm( + IntCC::Equal, + self.discriminant, + super::stack_switching_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT as i64, + ) + } + + pub fn is_absent<'a>( + &self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + builder.ins().icmp_imm( + IntCC::Equal, + self.discriminant, + super::stack_switching_environ::STACK_CHAIN_ABSENT_DISCRIMINANT as i64, + ) + } + + /// Return the two raw `ir::Value`s that represent this VMStackChain. + pub fn to_raw_parts(&self) -> [ir::Value; 2] { + [self.discriminant, self.payload] + } + + /// Construct a `Self` from two raw `ir::Value`s. + pub fn from_raw_parts(raw_data: [ir::Value; 2]) -> VMStackChain { + VMStackChain { + discriminant: raw_data[0], + payload: raw_data[1], + } + } + + /// Load a `VMStackChain` object from the given address. + pub fn load<'a>( + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + pointer: ir::Value, + initial_offset: i32, + pointer_type: ir::Type, + ) -> VMStackChain { + let memflags = ir::MemFlags::trusted(); + let mut offset = initial_offset; + let mut data = vec![]; + for _ in 0..2 { + data.push(builder.ins().load(pointer_type, memflags, pointer, offset)); + offset += pointer_type.bytes() as i32; + } + let data = <[ir::Value; 2]>::try_from(data).unwrap(); + Self::from_raw_parts(data) + } + + /// Store this `VMStackChain` object at the given address. + pub fn store<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + target_pointer: ir::Value, + initial_offset: i32, + ) { + let memflags = ir::MemFlags::trusted(); + let mut offset = initial_offset; + let data = self.to_raw_parts(); + + for value in data { + debug_assert_eq!(builder.func.dfg.value_type(value), env.pointer_type()); + builder.ins().store(memflags, value, target_pointer, offset); + offset += env.pointer_type().bytes() as i32; + } + } + + /// Use this only if you've already checked that `self` corresponds to a `VMStackChain::Continuation`. + pub fn unchecked_get_continuation<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + if cfg!(debug_assertions) { + let continuation_discriminant = + super::stack_switching_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT; + let is_continuation = builder.ins().icmp_imm( + IntCC::Equal, + self.discriminant, + continuation_discriminant as i64, + ); + emit_debug_assert!(env, builder, is_continuation); + } + self.payload + } + + /// Must only be called if `self` represents a `InitialStack` or + /// `Continuation` variant. Returns a pointer to the associated + /// `CommonStackInformation` object. + pub fn get_common_stack_information<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMCommonStackInformation { + self.assert_not_absent(env, builder); + + // `self` corresponds to a VMStackChain::InitialStack or + // VMStackChain::Continuation. + // In both cases, the payload is a pointer. + let address = self.payload; + + // `obj` is now a pointer to the beginning of either + // 1. A `VMContRef` struct (in the case of a + // VMStackChain::Continuation) + // 2. A CommonStackInformation struct (in the case of + // VMStackChain::InitialStack) + // + // Since a `VMContRef` starts with an (inlined) CommonStackInformation + // object at offset 0, we actually have in both cases that `ptr` is + // now the address of the beginning of a VMStackLimits object. + debug_assert_eq!(env.offsets.ptr.vmcontref_common_stack_information(), 0); + VMCommonStackInformation { address } + } + } + + impl VMCommonStackInformation { + fn get_state_ptr<'a>( + &self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let offset = _env.offsets.ptr.vmcommon_stack_information_state() as i64; + + builder.ins().iadd_imm(self.address, offset) + } + + fn get_stack_limits_ptr<'a>( + &self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let offset = _env.offsets.ptr.vmcommon_stack_information_limits() as i64; + + builder.ins().iadd_imm(self.address, offset) + } + + fn load_state<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let mem_flags = ir::MemFlags::trusted(); + let state_ptr = self.get_state_ptr(env, builder); + + builder.ins().load(I32, mem_flags, state_ptr, 0) + } + + fn set_state_no_payload<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + discriminant: u32, + ) { + let discriminant = builder.ins().iconst(I32, discriminant as i64); + emit_debug_println!( + env, + builder, + "setting state of CommonStackInformation {:p} to {}", + self.address, + discriminant + ); + + let mem_flags = ir::MemFlags::trusted(); + let state_ptr = self.get_state_ptr(env, builder); + + builder.ins().store(mem_flags, discriminant, state_ptr, 0); + } + + pub fn set_state_running<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = wasmtime_environ::stack_switching::STACK_STATE_RUNNING_DISCRIMINANT; + self.set_state_no_payload(env, builder, discriminant); + } + + pub fn set_state_parent<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = wasmtime_environ::stack_switching::STACK_STATE_PARENT_DISCRIMINANT; + self.set_state_no_payload(env, builder, discriminant); + } + + pub fn set_state_returned<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = wasmtime_environ::stack_switching::STACK_STATE_RETURNED_DISCRIMINANT; + self.set_state_no_payload(env, builder, discriminant); + } + + pub fn set_state_suspended<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = + wasmtime_environ::stack_switching::STACK_STATE_SUSPENDED_DISCRIMINANT; + self.set_state_no_payload(env, builder, discriminant); + } + + pub fn has_state_any_of<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + state_discriminants: &[u32], + ) -> ir::Value { + let actual_state = self.load_state(env, builder); + let zero = builder.ins().iconst(I8, 0); + let mut res = zero; + for state_discriminant in state_discriminants { + let eq = + builder + .ins() + .icmp_imm(IntCC::Equal, actual_state, *state_discriminant as i64); + res = builder.ins().bor(res, eq); + } + res + } + + pub fn has_state_returned<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + self.has_state_any_of( + env, + builder, + &[wasmtime_environ::stack_switching::STACK_STATE_RETURNED_DISCRIMINANT], + ) + } + + pub fn has_state_running<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + self.has_state_any_of( + env, + builder, + &[wasmtime_environ::stack_switching::STACK_STATE_RUNNING_DISCRIMINANT], + ) + } + + // pub fn has_state<'a>( + // &self, + // env: &mut crate::func_environ::FuncEnvironment<'a>, + // builder: &mut FunctionBuilder, + // state: super::stack_switching_environ::VMStackState, + // ) -> ir::Value { + // self.has_state_any_of(env, builder, &[state]) + // } + + /// Checks whether the `VMStackState` reflects that the stack has ever been + /// active (instead of just having been allocated, but never resumed). + pub fn was_invoked<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let actual_state = self.load_state(env, builder); + let allocated = wasmtime_environ::stack_switching::STACK_STATE_FRESH_DISCRIMINANT; + builder + .ins() + .icmp_imm(IntCC::NotEqual, actual_state, allocated as i64) + } + + pub fn get_handler_list<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + _builder: &mut FunctionBuilder, + ) -> VMHandlerList { + let offset = env.offsets.ptr.vmcommon_stack_information_handlers() as i32; + VMHandlerList::new(self.address, offset) + } + + pub fn get_first_switch_handler_index<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + // Field first_switch_handler_index has type u32 + let memflags = ir::MemFlags::trusted(); + let offset = + env.offsets + .ptr + .vmcommon_stack_information_first_switch_handler_index() as i32; + builder.ins().load(I32, memflags, self.address, offset) + } + + pub fn set_first_switch_handler_index<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + value: ir::Value, + ) { + // Field first_switch_handler_index has type u32 + let memflags = ir::MemFlags::trusted(); + let offset = + env.offsets + .ptr + .vmcommon_stack_information_first_switch_handler_index() as i32; + builder.ins().store(memflags, value, self.address, offset); + } + + /// Sets `last_wasm_entry_sp` and `stack_limit` fields in + /// `VMRuntimelimits` using the values from the `VMStackLimits` of this + /// object. + pub fn write_limits_to_vmcontext<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmruntime_limits_ptr: ir::Value, + ) { + let stack_limits_ptr = self.get_stack_limits_ptr(env, builder); + + let memflags = ir::MemFlags::trusted(); + + let mut copy_to_vm_runtime_limits = |our_offset, their_offset| { + let our_value = builder.ins().load( + env.pointer_type(), + memflags, + stack_limits_ptr, + i32::try_from(our_offset).unwrap(), + ); + builder.ins().store( + memflags, + our_value, + vmruntime_limits_ptr, + their_offset as i32, + ); + }; + + let pointer_size = u8::try_from(env.pointer_type().bytes()).unwrap(); + let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit() as i32; + let last_wasm_entry_fp_offset = + env.offsets.ptr.vmstack_limits_last_wasm_entry_fp() as i32; + copy_to_vm_runtime_limits( + stack_limit_offset, + pointer_size.vmstore_context_stack_limit(), + ); + copy_to_vm_runtime_limits( + last_wasm_entry_fp_offset, + pointer_size.vmstore_context_last_wasm_entry_fp(), + ); + } + + /// Overwrites the `last_wasm_entry_fp` field of the `VMStackLimits` + /// object in the `VMStackLimits` of this object by loading the corresponding + /// field from the `VMRuntimeLimits`. + /// If `load_stack_limit` is true, we do the same for the `stack_limit` + /// field. + pub fn load_limits_from_vmcontext<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmruntime_limits_ptr: ir::Value, + load_stack_limit: bool, + ) { + let stack_limits_ptr = self.get_stack_limits_ptr(env, builder); + + let memflags = ir::MemFlags::trusted(); + let pointer_size = u8::try_from(env.pointer_type().bytes()).unwrap(); + + let mut copy = |runtime_limits_offset, stack_limits_offset| { + let from_vm_runtime_limits = builder.ins().load( + env.pointer_type(), + memflags, + vmruntime_limits_ptr, + runtime_limits_offset, + ); + builder.ins().store( + memflags, + from_vm_runtime_limits, + stack_limits_ptr, + i32::try_from(stack_limits_offset).unwrap(), + ); + }; + + let last_wasm_entry_fp_offset = + env.offsets.ptr.vmstack_limits_last_wasm_entry_fp() as i32; + copy( + pointer_size.vmstore_context_last_wasm_entry_fp(), + last_wasm_entry_fp_offset, + ); + + if load_stack_limit { + let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit() as i32; + copy( + pointer_size.vmstore_context_stack_limit(), + stack_limit_offset, + ); + } + } + } + + impl VMContinuationStack { + /// The parameter is NOT the "top of stack" address of the stack itself. In line + /// with how the (runtime) `FiberStack` type works, this is a pointer to + /// the TOS address. + pub fn new(tos_ptr: ir::Value) -> Self { + Self { tos_ptr } + } + + fn load_top_of_stack<'a>( + &self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let mem_flags = ir::MemFlags::trusted(); + builder.ins().load(I64, mem_flags, self.tos_ptr, 0) + } + + /// Returns address of the control context stored in the stack memory, + /// as used by stack_switch instructions. + pub fn load_control_context<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let tos = self.load_top_of_stack(env, builder); + // Control context begins 24 bytes below top of stack (see unix.rs) + builder.ins().iadd_imm(tos, -0x18) + } + } +} + +use helpers::VMStackChain; +use stack_switching_environ::{ + CONTROL_EFFECT_RESUME_DISCRIMINANT, CONTROL_EFFECT_SWITCH_DISCRIMINANT, +}; +use stack_switching_helpers as helpers; + +/// Stores the given arguments in the appropriate `VMPayloads` object in the +/// continuation. If the continuation was never invoked, use the `args` object. +/// Otherwise, use the `values` object. +pub(crate) fn vmcontref_store_payloads<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + values: &[ir::Value], + contref: ir::Value, +) { + let count = + i32::try_from(values.len()).expect("Number of stack switching payloads should fit in i32"); + if values.len() > 0 { + let use_args_block = builder.create_block(); + let use_payloads_block = builder.create_block(); + let store_data_block = builder.create_block(); + builder.append_block_param(store_data_block, env.pointer_type()); + + let co = helpers::VMContRef::new(contref); + let csi = co.common_stack_information(env, builder); + let was_invoked = csi.was_invoked(env, builder); + builder + .ins() + .brif(was_invoked, use_payloads_block, &[], use_args_block, &[]); + + { + builder.switch_to_block(use_args_block); + builder.seal_block(use_args_block); + + let args = co.args(env, builder); + let ptr = args.occupy_next_slots(env, builder, count); + + builder + .ins() + .jump(store_data_block, &[BlockArg::Value(ptr)]); + } + + { + builder.switch_to_block(use_payloads_block); + builder.seal_block(use_payloads_block); + + let payloads = co.values(env, builder); + + // This also checks that the buffer is large enough to hold + // `values.len()` more elements. + let ptr = payloads.occupy_next_slots(env, builder, count); + builder + .ins() + .jump(store_data_block, &[BlockArg::Value(ptr)]); + } + + { + builder.switch_to_block(store_data_block); + builder.seal_block(store_data_block); + + let ptr = builder.block_params(store_data_block)[0]; + + // Store the values. + let memflags = ir::MemFlags::trusted(); + let mut offset = 0; + for value in values { + builder.ins().store(memflags, *value, ptr, offset); + offset += env.offsets.ptr.maximum_value_size() as i32; + } + } + } +} + +pub(crate) fn tag_address<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + index: u32, +) -> ir::Value { + let vmctx = env.vmctx_val(&mut builder.cursor()); + let tag_index = wasmtime_environ::TagIndex::from_u32(index); + let pointer_type = env.pointer_type(); + if let Some(def_index) = env.module.defined_tag_index(tag_index) { + let offset = i32::try_from(env.offsets.vmctx_vmtag_definition(def_index)).unwrap(); + builder.ins().iadd_imm(vmctx, offset as i64) + } else { + let offset = i32::try_from(env.offsets.vmctx_vmtag_import_from(tag_index)).unwrap(); + builder.ins().load( + pointer_type, + ir::MemFlags::trusted().with_readonly(), + vmctx, + ir::immediates::Offset32::new(offset), + ) + } +} + +/// Returns the stack chain saved in the given `VMContext`. Note that the +/// head of the list is the actively running stack (initial stack or +/// continuation). +pub fn vmctx_load_stack_chain<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmctx: ir::Value, +) -> VMStackChain { + let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain() as i32; + + // First we need to get the `VMStoreContext`. + let vm_store_context_offset = env.offsets.ptr.vmctx_store_context(); + let vm_store_context = builder.ins().load( + env.pointer_type(), + MemFlags::trusted(), + vmctx, + vm_store_context_offset, + ); + + VMStackChain::load( + env, + builder, + vm_store_context, + stack_chain_offset, + env.pointer_type(), + ) +} + +/// Stores the given stack chain saved in the `VMContext`, overwriting the +/// exsiting one. +pub fn vmctx_store_stack_chain<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmctx: ir::Value, + stack_chain: &VMStackChain, +) { + let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain() as i32; + + // First we need to get the `VMStoreContext`. + let vm_store_context_offset = env.offsets.ptr.vmctx_store_context(); + let vm_store_context = builder.ins().load( + env.pointer_type(), + MemFlags::trusted(), + vmctx, + vm_store_context_offset, + ); + + stack_chain.store(env, builder, vm_store_context, stack_chain_offset) +} + +/// Similar to `vmctx_store_stack_chain`, but instead of storing an arbitrary +/// `VMStackChain`, stores VMStackChain::Continuation(contref)`. +pub fn vmctx_set_active_continuation<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmctx: ir::Value, + contref: ir::Value, +) { + let chain = VMStackChain::from_continuation(env, builder, contref); + vmctx_store_stack_chain(env, builder, vmctx, &chain) +} + +pub fn vmctx_load_vm_runtime_limits_ptr<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmctx: ir::Value, +) -> ir::Value { + let pointer_type = env.pointer_type(); + let offset = i32::from(env.offsets.ptr.vmctx_store_context()); + + // The *pointer* to the VMRuntimeLimits does not change within the + // same function, allowing us to set the `read_only` flag. + let flags = ir::MemFlags::trusted().with_readonly(); + + builder.ins().load(pointer_type, flags, vmctx, offset) +} + +/// This function generates code that searches for a handler for `tag_address`, +/// which must be a `*mut VMTagDefinition`. The search walks up the chain of +/// continuations beginning at `start`. +/// +/// The flag `search_suspend_handlers` determines whether we search for a +/// suspend or switch handler. Concretely, this influences which part of each +/// handler list we will search. +/// +/// We trap if no handler was found. +/// +/// The returned values are: +/// 1. The stack (continuation or initial stack, represented as a VMStackChain) in +/// whose handler list we found the tag (i.e., the stack that performed the +/// resume instruction that installed handler for the tag). +/// 2. The continuation whose parent is the stack mentioned in 1. +/// 3. The index of the handler in the handler list. +/// +/// In pseudo-code, the generated code's behavior can be expressed as +/// follows: +/// +/// chain_link = start +/// while !chain_link.is_initial_stack() { +/// contref = chain_link.get_contref() +/// parent_link = contref.parent +/// parent_csi = parent_link.get_common_stack_information(); +/// handlers = parent_csi.handlers; +/// (begin_range, end_range) = if search_suspend_handlers { +/// (0, parent_csi.first_switch_handler_index) +/// } else { +/// (parent_csi.first_switch_handler_index, handlers.length) +/// }; +/// for index in begin_range..end_range { +/// if handlers[index] == tag_address { +/// goto on_match(contref, index) +/// } +/// } +/// chain_link = parent_link +/// } +/// trap(unhandled_tag) +/// +/// on_match(conref : VMContRef, handler_index : u32) +/// ... execution continues here here ... +/// +fn search_handler<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + start: &helpers::VMStackChain, + tag_address: ir::Value, + search_suspend_handlers: bool, +) -> (VMStackChain, ir::Value, ir::Value) { + let handle_link = builder.create_block(); + let begin_search_handler_list = builder.create_block(); + let try_index = builder.create_block(); + let compare_tags = builder.create_block(); + let on_match = builder.create_block(); + let on_no_match = builder.create_block(); + let block_args = start.to_raw_parts().map(|v| BlockArg::Value(v)); + + // Terminate previous block: + builder.ins().jump(handle_link, &block_args); + + // Block handle_link + let chain_link = { + builder.append_block_param(handle_link, env.pointer_type()); + builder.append_block_param(handle_link, env.pointer_type()); + builder.switch_to_block(handle_link); + + let raw_parts = builder.block_params(handle_link); + let chain_link = helpers::VMStackChain::from_raw_parts([raw_parts[0], raw_parts[1]]); + let is_initial_stack = chain_link.is_initial_stack(env, builder); + builder.ins().brif( + is_initial_stack, + on_no_match, + &[], + begin_search_handler_list, + &[], + ); + chain_link + }; + + // Block begin_search_handler_list + let (contref, parent_link, handler_list_data_ptr, end_range) = { + builder.switch_to_block(begin_search_handler_list); + let contref = chain_link.unchecked_get_continuation(env, builder); + let contref = helpers::VMContRef::new(contref); + + let parent_link = contref.get_parent_stack_chain(env, builder); + + emit_debug_println!( + env, + builder, + "[search_handler] beginning search in parent of continuation {:p}", + contref.address + ); + + let parent_csi = parent_link.get_common_stack_information(env, builder); + + let handlers = parent_csi.get_handler_list(env, builder); + let handler_list_data_ptr = handlers.get_data(env, builder); + + let first_switch_handler_index = parent_csi.get_first_switch_handler_index(env, builder); + + // Note that these indices are inclusive-exclusive, i.e. [begin_range, end_range). + let (begin_range, end_range) = if search_suspend_handlers { + let zero = builder.ins().iconst(I32, 0); + if cfg!(debug_assertions) { + let length = handlers.get_length(env, builder); + emit_debug_assert_ule!(env, builder, first_switch_handler_index, length); + } + (zero, first_switch_handler_index) + } else { + let length = handlers.get_length(env, builder); + (first_switch_handler_index, length) + }; + + builder + .ins() + .jump(try_index, &[BlockArg::Value(begin_range)]); + + (contref, parent_link, handler_list_data_ptr, end_range) + }; + + // Block try_index + let index = { + builder.append_block_param(try_index, I32); + builder.switch_to_block(try_index); + let index = builder.block_params(try_index)[0]; + + let in_bounds = builder + .ins() + .icmp(IntCC::UnsignedLessThan, index, end_range); + let block_args = parent_link.to_raw_parts().map(|v| BlockArg::Value(v)); + builder + .ins() + .brif(in_bounds, compare_tags, &[], handle_link, &block_args); + index + }; + + // Block compare_tags + { + builder.switch_to_block(compare_tags); + + let base = handler_list_data_ptr; + let entry_size = std::mem::size_of::<*mut u8>(); + let offset = builder.ins().imul_imm(index, entry_size as i64); + let offset = builder.ins().uextend(I64, offset); + let entry_address = builder.ins().iadd(base, offset); + + let memflags = ir::MemFlags::trusted(); + + let handled_tag = builder + .ins() + .load(env.pointer_type(), memflags, entry_address, 0); + + let tags_match = builder.ins().icmp(IntCC::Equal, handled_tag, tag_address); + let incremented_index = builder.ins().iadd_imm(index, 1); + builder.ins().brif( + tags_match, + on_match, + &[], + try_index, + &[BlockArg::Value(incremented_index)], + ); + } + + // Block on_no_match + { + builder.switch_to_block(on_no_match); + builder.set_cold_block(on_no_match); + builder.ins().trap(crate::TRAP_UNHANDLED_TAG); + } + + builder.seal_block(handle_link); + builder.seal_block(begin_search_handler_list); + builder.seal_block(try_index); + builder.seal_block(compare_tags); + builder.seal_block(on_match); + builder.seal_block(on_no_match); + + // final block: on_match + builder.switch_to_block(on_match); + + emit_debug_println!( + env, + builder, + "[search_handler] found handler at stack chain ({}, {:p}), whose child continuation is {:p}, index is {}", + parent_link.to_raw_parts()[0], + parent_link.to_raw_parts()[1], + contref.address, + index + ); + + (parent_link, contref.address, index) +} + +pub(crate) fn translate_cont_bind<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + contobj: ir::Value, + args: &[ir::Value], +) -> ir::Value { + let (witness, contref) = fatpointer::deconstruct(env, &mut builder.cursor(), contobj); + + // The typing rules for cont.bind allow a null reference to be passed to it. + builder.ins().trapz(contref, crate::TRAP_NULL_REFERENCE); + + let mut vmcontref = helpers::VMContRef::new(contref); + let revision = vmcontref.get_revision(env, builder); + let evidence = builder.ins().icmp(IntCC::Equal, witness, revision); + emit_debug_println!( + env, + builder, + "[cont_bind] witness = {}, revision = {}, evidence = {}", + witness, + revision, + evidence + ); + builder + .ins() + .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); + + vmcontref_store_payloads(env, builder, args, contref); + + let revision = vmcontref.incr_revision(env, builder, revision); + emit_debug_println!(env, builder, "new revision = {}", revision); + let contobj = fatpointer::construct(env, &mut builder.cursor(), revision, contref); + emit_debug_println!(env, builder, "[cont_bind] contref = {:p}", contref); + contobj +} + +pub(crate) fn translate_cont_new<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + func: ir::Value, + arg_types: &[WasmValType], + return_types: &[WasmValType], +) -> WasmResult { + // The typing rules for cont.new allow a null reference to be passed to it. + builder.ins().trapz(func, crate::TRAP_NULL_REFERENCE); + + let nargs = builder.ins().iconst(I32, arg_types.len() as i64); + let nreturns = builder.ins().iconst(I32, return_types.len() as i64); + + let cont_new_func = env.builtin_functions.cont_new(&mut builder.func); + let vmctx = env.vmctx_val(&mut builder.cursor()); + let call_inst = builder + .ins() + .call(cont_new_func, &[vmctx, func, nargs, nreturns]); + let contref = *builder.func.dfg.inst_results(call_inst).first().unwrap(); + + let tag = helpers::VMContRef::new(contref).get_revision(env, builder); + let contobj = fatpointer::construct(env, &mut builder.cursor(), tag, contref); + emit_debug_println!(env, builder, "[cont_new] contref = {:p}", contref); + Ok(contobj) +} + +pub(crate) fn translate_resume<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + type_index: u32, + resume_contobj: ir::Value, + resume_args: &[ir::Value], + resumetable: &[(u32, Option)], +) -> WasmResult> { + // The resume instruction is the most involved instruction to + // compile as it is responsible for both continuation application + // and control tag dispatch. + // + // Here we translate a resume instruction into several basic + // blocks as follows: + // + // previous block + // | + // | + // resume_block + // / \ + // / \ + // | | + // return_block | + // suspend block + // | + // dispatch block + // + // * resume_block handles continuation arguments and performs + // actual stack switch. On ordinary return from resume, it jumps + // to the `return_block`, whereas on suspension it jumps to the + // `suspend_block`. + // * suspend_block is used on suspension, jumps onward to + // `dispatch_block`. + // * dispatch_block uses a jump table to dispatch to actual + // user-defined handler blocks, based on the handler index + // provided on suspension. Note that we do not jump to the + // handler blocks directly. Instead, each handler block has a + // corresponding premable block, which we jump to in order to + // reach a particular handler block. The preamble block prepares + // the arguments and continuation object to be passed to the + // actual handler block. + // + let resume_block = builder.create_block(); + let return_block = builder.create_block(); + let suspend_block = builder.create_block(); + let dispatch_block = builder.create_block(); + + let vmctx = env.vmctx_val(&mut builder.cursor()); + + // Split the resumetable into suspend handlers (each represented by the tag + // index and handler block) and the switch handlers (represented just by the + // tag index). Note that we currently don't remove duplicate tags. + let (suspend_handlers, switch_tags): (Vec<(u32, Block)>, Vec) = resumetable + .iter() + .partition_map(|(tag_index, block_opt)| match block_opt { + Some(block) => Either::Left((*tag_index, *block)), + None => Either::Right(*tag_index), + }); + + // Technically, there is no need to have a dedicated resume block, we could + // just put all of its contents into the current block. + builder.ins().jump(resume_block, &[]); + + // Resume block: actually resume the continuation chain ending at `resume_contref`. + let (resume_result, vm_runtime_limits_ptr, original_stack_chain, new_stack_chain) = { + builder.switch_to_block(resume_block); + builder.seal_block(resume_block); + + let (witness, resume_contref) = + fatpointer::deconstruct(env, &mut builder.cursor(), resume_contobj); + + // The typing rules for resume allow a null reference to be passed to it. + builder + .ins() + .trapz(resume_contref, crate::TRAP_NULL_REFERENCE); + + let mut vmcontref = helpers::VMContRef::new(resume_contref); + + let revision = vmcontref.get_revision(env, builder); + let evidence = builder.ins().icmp(IntCC::Equal, revision, witness); + emit_debug_println!( + env, + builder, + "[resume] resume_contref = {:p} witness = {}, revision = {}, evidence = {}", + resume_contref, + witness, + revision, + evidence + ); + builder + .ins() + .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); + let next_revision = vmcontref.incr_revision(env, builder, revision); + emit_debug_println!(env, builder, "[resume] new revision = {}", next_revision); + + if cfg!(debug_assertions) { + // This should be impossible due to the linearity check. + let zero = builder.ins().iconst(I8, 0); + let csi = vmcontref.common_stack_information(env, builder); + let has_returned = csi.has_state_returned(env, builder); + emit_debug_assert_eq!(env, builder, has_returned, zero); + } + + if resume_args.len() > 0 { + // We store the arguments in the `VMContRef` to be resumed. + vmcontref_store_payloads(env, builder, resume_args, resume_contref); + } + + // Splice together stack chains: + // Connect the end of the chain starting at `resume_contref` to the currently active chain. + let mut last_ancestor = helpers::VMContRef::new(vmcontref.get_last_ancestor(env, builder)); + + // Make the currently running continuation (if any) the parent of the one we are about to resume. + let original_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + original_stack_chain.assert_not_absent(env, builder); + if cfg!(debug_assertions) { + // The continuation we are about to resume should have its chain broken up at last_ancestor. + let last_ancestor_chain = last_ancestor.get_parent_stack_chain(env, builder); + let is_absent = last_ancestor_chain.is_absent(env, builder); + emit_debug_assert!(env, builder, is_absent); + } + last_ancestor.set_parent_stack_chain(env, builder, &original_stack_chain); + + emit_debug_println!( + env, + builder, + "[resume] spliced together stack chains: parent of {:p} (last ancestor of {:p}) is now pointing to ({}, {:p})", + last_ancestor.address, + vmcontref.address, + original_stack_chain.to_raw_parts()[0], + original_stack_chain.to_raw_parts()[1] + ); + + // Just for consistency: `vmcontref` is about to get state Running, so let's zero out its last_ancestor field. + let zero = builder.ins().iconst(env.pointer_type(), 0); + vmcontref.set_last_ancestor(env, builder, zero); + + // We mark `resume_contref` as the currently running one + vmctx_set_active_continuation(env, builder, vmctx, resume_contref); + + // Note that the resume_contref libcall a few lines further below + // manipulates the stack limits as follows: + // 1. Copy stack_limit, last_wasm_entry_sp and last_wasm_exit* values from + // VMRuntimeLimits into the currently active continuation (i.e., the + // one that will become the parent of the to-be-resumed one) + // + // 2. Copy `stack_limit` and `last_wasm_entry_sp` in the + // `VMStackLimits` of `resume_contref` into the `VMRuntimeLimits`. + // + // See the comment on `stack_switching_environ::VMStackChain` for a + // description of the invariants that we maintain for the various stack + // limits. + + // `resume_contref` is now active, and its parent is suspended. + let resume_contref = helpers::VMContRef::new(resume_contref); + let resume_csi = resume_contref.common_stack_information(env, builder); + let parent_csi = original_stack_chain.get_common_stack_information(env, builder); + resume_csi.set_state_running(env, builder); + parent_csi.set_state_parent(env, builder); + + // We update the `VMStackLimits` of the parent of the continuation to be resumed + // as well as the `VMRuntimeLimits`. + // See the comment on `stack_switching_environ::VMStackChain` for a description + // of the invariants that we maintain for the various stack limits. + let vm_runtime_limits_ptr = vmctx_load_vm_runtime_limits_ptr(env, builder, vmctx); + parent_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, true); + resume_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); + + // Install handlers in (soon to be) parent's VMHandlerList: + // Let the i-th handler clause be (on $tag $block). + // Then the i-th entry of the VMHandlerList will be the address of $tag. + let handler_list = parent_csi.get_handler_list(env, builder); + + if resumetable.len() > 0 { + // Total number of handlers (suspend and switch). + let handler_count = u32::try_from(resumetable.len()).unwrap(); + // Populate the Array's data ptr with a pointer to a sufficiently + // large area on this stack. + env.stack_switching_handler_list_buffer = + Some(handler_list.allocate_or_reuse_stack_slot( + env, + builder, + handler_count, + env.stack_switching_handler_list_buffer, + )); + + let suspend_handler_count = suspend_handlers.len(); + + // All handlers, represented by the indices of the tags they handle. + // All the suspend handlers come first, followed by all the switch handlers. + let all_handlers = suspend_handlers + .iter() + .map(|(tag_index, _block)| *tag_index) + .chain(switch_tags); + + // Translate all tag indices to tag addresses (i.e., the corresponding *mut VMTagDefinition). + let all_tag_addresses: Vec = all_handlers + .map(|tag_index| tag_address(env, builder, tag_index)) + .collect(); + + // Store all tag addresess in the handler list. + handler_list.store_data_entries(env, builder, &all_tag_addresses, false); + + // To enable distinguishing switch and suspend handlers when searching the handler list: + // Store at which index the switch handlers start. + let first_switch_handler_index = + builder.ins().iconst(I32, suspend_handler_count as i64); + parent_csi.set_first_switch_handler_index(env, builder, first_switch_handler_index); + } + + let resume_payload = ControlEffect::encode_resume(env, builder).to_u64(); + + // Note that the control context we use for switching is not the one in + // (the stack of) resume_contref, but in (the stack of) last_ancestor! + let fiber_stack = last_ancestor.get_fiber_stack(env, builder); + let control_context_ptr = fiber_stack.load_control_context(env, builder); + + emit_debug_println!( + env, + builder, + "[resume] about to execute stack_switch, control_context_ptr is {:p}", + control_context_ptr + ); + + let result = + builder + .ins() + .stack_switch(control_context_ptr, control_context_ptr, resume_payload); + + emit_debug_println!( + env, + builder, + "[resume] continuing after stack_switch in frame with parent_stack_chain ({}, {:p}), result is {:p}", + original_stack_chain.to_raw_parts()[0], + original_stack_chain.to_raw_parts()[1], + result + ); + + // At this point we know nothing about the continuation that just + // suspended or returned. In particular, it does not have to be what we + // called `resume_contref` earlier on. We must reload the information + // about the now active continuation from the VMContext. + let new_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + + // Now the parent contref (or initial stack) is active again + vmctx_store_stack_chain(env, builder, vmctx, &original_stack_chain); + parent_csi.set_state_running(env, builder); + + // Just for consistency: Clear the handler list. + handler_list.clear(env, builder, true); + parent_csi.set_first_switch_handler_index(env, builder, zero); + + // Extract the result and signal bit. + let result = ControlEffect::from_u64(result); + let signal = result.signal(env, builder); + + emit_debug_println!( + env, + builder, + "[resume] in resume block, signal is {}", + signal + ); + + // Jump to the return block if the result signal is 0, otherwise jump to + // the suspend block. + builder + .ins() + .brif(signal, suspend_block, &[], return_block, &[]); + + ( + result, + vm_runtime_limits_ptr, + original_stack_chain, + new_stack_chain, + ) + }; + + // The suspend block: Only used when we suspended, not for returns. + // Here we extract the index of the handler to use. + let (handler_index, suspended_contref, suspended_contobj) = { + builder.switch_to_block(suspend_block); + builder.seal_block(suspend_block); + + let suspended_continuation = new_stack_chain.unchecked_get_continuation(env, builder); + let mut suspended_continuation = helpers::VMContRef::new(suspended_continuation); + let suspended_csi = suspended_continuation.common_stack_information(env, builder); + + // Note that at the suspend site, we already + // 1. Set the state of suspended_continuation to Suspended + // 2. Set suspended_continuation.last_ancestor + // 3. Broke the continuation chain at suspended_continuation.last_ancestor + + // We store parts of the VMRuntimeLimits into the continuation that just suspended. + suspended_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false); + + // Afterwards (!), restore parts of the VMRuntimeLimits from the + // parent of the suspended continuation (which is now active). + let parent_csi = original_stack_chain.get_common_stack_information(env, builder); + parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); + + // Extract the handler index + let handler_index = ControlEffect::handler_index(resume_result, env, builder); + + let revision = suspended_continuation.get_revision(env, builder); + let suspended_contobj = fatpointer::construct( + env, + &mut builder.cursor(), + revision, + suspended_continuation.address, + ); + + emit_debug_println!( + env, + builder, + "[resume] in suspend block, handler index is {}, new continuation is {:p}, with existing revision {}", + handler_index, + suspended_continuation.address, + revision + ); + + // We need to terminate this block before being allowed to switch to + // another one. + builder.ins().jump(dispatch_block, &[]); + + (handler_index, suspended_continuation, suspended_contobj) + }; + + // For technical reasons, the jump table needs to have a default + // block. In our case, it should be unreachable, since the handler + // index we dispatch on should correspond to a an actual handler + // block in the jump table. + let jt_default_block = builder.create_block(); + { + builder.switch_to_block(jt_default_block); + builder.set_cold_block(jt_default_block); + + builder.ins().trap(crate::TRAP_UNREACHABLE); + } + + // We create a preamble block for each of the actual handler blocks: It + // reads the necessary arguments and passes them to the actual handler + // block, together with the continuation object. + let target_preamble_blocks = { + let mut preamble_blocks = vec![]; + + for &(handle_tag, target_block) in &suspend_handlers { + let preamble_block = builder.create_block(); + preamble_blocks.push(preamble_block); + builder.switch_to_block(preamble_block); + + let param_types = env.tag_params(TagIndex::from_u32(handle_tag)); + let param_types: Vec = param_types + .iter() + .map(|wty| crate::value_type(env.isa, *wty)) + .collect(); + + let values = suspended_contref.values(env, builder); + let mut suspend_args: Vec = values + .load_data_entries(env, builder, ¶m_types) + .into_iter() + .map(|v| BlockArg::Value(v)) + .collect(); + + // At the suspend site, we store the suspend args in the the + // `values` buffer of the VMContRef that was active at the time that + // the suspend instruction was performed. + suspend_args.push(BlockArg::Value(suspended_contobj)); + + // We clear the suspend args. This is mostly for consistency. Note + // that we don't zero out the data buffer, we still need it for the + + values.clear(env, builder, false); + + builder.ins().jump(target_block, &suspend_args); + } + + preamble_blocks + }; + + // Dispatch block. All it does is jump to the right premable block based on + // the handler index. + { + builder.switch_to_block(dispatch_block); + builder.seal_block(dispatch_block); + + let default_bc = builder.func.dfg.block_call(jt_default_block, &[]); + + let adapter_bcs: Vec = target_preamble_blocks + .iter() + .map(|b| builder.func.dfg.block_call(*b, &[])) + .collect(); + + let jt_data = JumpTableData::new(default_bc, &adapter_bcs); + let jt = builder.create_jump_table(jt_data); + + builder.ins().br_table(handler_index, jt); + + for preamble_block in target_preamble_blocks { + builder.seal_block(preamble_block); + } + builder.seal_block(jt_default_block); + } + + // Return block: Jumped to by resume block if continuation + // returned normally. + { + builder.switch_to_block(return_block); + builder.seal_block(return_block); + + // If we got a return signal, a continuation must have been running. + let returned_contref = new_stack_chain.unchecked_get_continuation(env, builder); + let returned_contref = helpers::VMContRef::new(returned_contref); + + // Restore parts of the VMRuntimeLimits from the parent of the + // returned continuation (which is now active). + let parent_csi = original_stack_chain.get_common_stack_information(env, builder); + parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); + + let returned_csi = returned_contref.common_stack_information(env, builder); + returned_csi.set_state_returned(env, builder); + + // Load the values returned by the continuation. + let return_types: Vec<_> = env + .continuation_returns(TypeIndex::from_u32(type_index)) + .iter() + .map(|ty| crate::value_type(env.isa, *ty)) + .collect(); + let payloads = returned_contref.args(env, builder); + let return_values = payloads.load_data_entries(env, builder, &return_types); + payloads.clear(env, builder, true); + + Ok(return_values) + } +} + +pub(crate) fn translate_suspend<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + tag_index: u32, + suspend_args: &[ir::Value], + tag_return_types: &[ir::Type], +) -> Vec { + let tag_addr = tag_address(env, builder, tag_index); + emit_debug_println!(env, builder, "[suspend] suspending with tag {:p}", tag_addr); + + let vmctx = env.vmctx_val(&mut builder.cursor()); + let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + + let (_, end_of_chain_contref, handler_index) = + search_handler(env, builder, &active_stack_chain, tag_addr, true); + + emit_debug_println!( + env, + builder, + "[suspend] found handler: end of chain contref is {:p}, handler index is {}", + end_of_chain_contref, + handler_index + ); + + // If we get here, the search_handler logic succeeded (i.e., did not trap). + // Thus, there is at least one parent, so we are not on the initial stack. + // Can therefore extract continuation directly. + let active_contref = active_stack_chain.unchecked_get_continuation(env, builder); + let active_contref = helpers::VMContRef::new(active_contref); + let mut end_of_chain_contref = helpers::VMContRef::new(end_of_chain_contref); + + active_contref.set_last_ancestor(env, builder, end_of_chain_contref.address); + + // In the active_contref's `values` buffer, stack-allocate enough room so that we can + // later store the following: + // 1. The suspend arguments + // 2. Afterwards, the tag return values + let values = active_contref.values(env, builder); + let required_capacity = + u32::try_from(std::cmp::max(suspend_args.len(), tag_return_types.len())) + .expect("Number of stack switching payloads should fit in u32"); + + if required_capacity > 0 { + env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot( + env, + builder, + required_capacity, + env.stack_switching_values_buffer, + )); + } + + if suspend_args.len() > 0 { + values.store_data_entries(env, builder, suspend_args, true) + } + + // Set current continuation to suspended and break up handler chain. + let active_contref_csi = active_contref.common_stack_information(env, builder); + if cfg!(debug_assertions) { + let is_running = active_contref_csi.has_state_running(env, builder); + emit_debug_assert!(env, builder, is_running); + } + + active_contref_csi.set_state_suspended(env, builder); + let absent_chain_link = VMStackChain::absent(env, builder); + end_of_chain_contref.set_parent_stack_chain(env, builder, &absent_chain_link); + + let suspend_payload = ControlEffect::encode_suspend(env, builder, handler_index).to_u64(); + + // Note that the control context we use for switching is the one + // at the end of the chain, not the one in active_contref! + // This also means that stack_switch saves the information about + // the current stack in the control context located in the stack + // of end_of_chain_contref. + let fiber_stack = end_of_chain_contref.get_fiber_stack(env, builder); + let control_context_ptr = fiber_stack.load_control_context(env, builder); + + builder + .ins() + .stack_switch(control_context_ptr, control_context_ptr, suspend_payload); + + // The return values of the suspend instruction are the tag return values, saved in the `args` buffer. + let values = active_contref.values(env, builder); + let return_values = values.load_data_entries(env, builder, tag_return_types); + // We effectively consume the values and discard the stack allocated buffer. + values.clear(env, builder, true); + + return_values +} + +pub(crate) fn translate_switch<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + tag_index: u32, + switchee_contobj: ir::Value, + switch_args: &[ir::Value], + return_types: &[ir::Type], +) -> WasmResult> { + let vmctx = env.vmctx_val(&mut builder.cursor()); + + // Check and increment revision on switchee continuation object (i.e., the + // one being switched to). Logically, the switchee continuation extends from + // `switchee_contref` to `switchee_contref.last_ancestor` (i.e., the end of + // the parent chain starting at `switchee_contref`). + let switchee_contref = { + let (witness, target_contref) = + fatpointer::deconstruct(env, &mut builder.cursor(), switchee_contobj); + + // The typing rules for switch allow a null reference to be passed to it. + builder + .ins() + .trapz(target_contref, crate::TRAP_NULL_REFERENCE); + + let mut target_contref = helpers::VMContRef::new(target_contref); + + let revision = target_contref.get_revision(env, builder); + let evidence = builder.ins().icmp(IntCC::Equal, revision, witness); + emit_debug_println!( + env, + builder, + "[switch] target_contref = {:p} witness = {}, revision = {}, evidence = {}", + target_contref.address, + witness, + revision, + evidence + ); + builder + .ins() + .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); + let _next_revision = target_contref.incr_revision(env, builder, revision); + target_contref + }; + + // We create the "switcher continuation" (i.e., the one executing switch) + // from the current execution context: Logically, it extends from the + // continuation reference executing `switch` (subsequently called + // `switcher_contref`) to the immediate child (called + // `switcher_contref_last_ancestor`) of the stack with the corresponding + // handler (saved in `handler_stack_chain`). + let ( + switcher_contref, + switcher_contobj, + switcher_contref_last_ancestor, + handler_stack_chain, + vm_runtime_limits_ptr, + ) = { + let tag_addr = tag_address(env, builder, tag_index); + let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + let (handler_stack_chain, last_ancestor, _handler_index) = + search_handler(env, builder, &active_stack_chain, tag_addr, false); + let mut last_ancestor = helpers::VMContRef::new(last_ancestor); + + // If we get here, the search_handler logic succeeded (i.e., did not trap). + // Thus, there is at least one parent, so we are not on the initial stack. + // Can therefore extract continuation directly. + let switcher_contref = active_stack_chain.unchecked_get_continuation(env, builder); + let mut switcher_contref = helpers::VMContRef::new(switcher_contref); + + switcher_contref.set_last_ancestor(env, builder, last_ancestor.address); + + // In the switcher_contref's `values` buffer, stack-allocate enough room so that we can + // later store `tag_return_types.len()` when resuming the continuation. + let values = switcher_contref.values(env, builder); + let required_capacity = u32::try_from(return_types.len()).unwrap(); + if required_capacity > 0 { + env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot( + env, + builder, + required_capacity, + env.stack_switching_values_buffer, + )); + } + + let switcher_contref_csi = switcher_contref.common_stack_information(env, builder); + emit_debug_assert!( + env, + builder, + switcher_contref_csi.has_state_running(env, builder) + ); + switcher_contref_csi.set_state_suspended(env, builder); + // We break off `switcher_contref` from the chain of active + // continuations, by separating the link between `last_ancestor` and its + // parent stack. + let absent = VMStackChain::absent(env, builder); + last_ancestor.set_parent_stack_chain(env, builder, &absent); + + // Load current runtime limits from `VMContext` and store in the + // switcher continuation. + let vm_runtime_limits_ptr = vmctx_load_vm_runtime_limits_ptr(env, builder, vmctx); + switcher_contref_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false); + + let revision = switcher_contref.get_revision(env, builder); + let new_contobj = fatpointer::construct( + env, + &mut builder.cursor(), + revision, + switcher_contref.address, + ); + + emit_debug_println!( + env, + builder, + "[switch] created new contref = {:p}, revision = {}", + switcher_contref.address, + revision + ); + + ( + switcher_contref, + new_contobj, + last_ancestor, + handler_stack_chain, + vm_runtime_limits_ptr, + ) + }; + + // Prepare switchee continuation: + // - Store "ordinary" switch arguments as well as the contobj just + // synthesized from the current context (i.e., `switcher_contobj`) in the + // switchee continuation's payload buffer. + // - Splice switchee's continuation chain with handler stack to form new + // overall chain of active continuations. + let (switchee_contref_csi, switchee_contref_last_ancestor) = { + let mut combined_payloads = switch_args.to_vec(); + combined_payloads.push(switcher_contobj); + vmcontref_store_payloads(env, builder, &combined_payloads, switchee_contref.address); + + let switchee_contref_csi = switchee_contref.common_stack_information(env, builder); + + emit_debug_assert!( + env, + builder, + switchee_contref_csi.has_state_any_of( + env, + builder, + &[ + wasmtime_environ::stack_switching::STACK_STATE_FRESH_DISCRIMINANT, + wasmtime_environ::stack_switching::STACK_STATE_SUSPENDED_DISCRIMINANT + ] + ) + ); + switchee_contref_csi.set_state_running(env, builder); + + let switchee_contref_last_ancestor = switchee_contref.get_last_ancestor(env, builder); + let mut switchee_contref_last_ancestor = + helpers::VMContRef::new(switchee_contref_last_ancestor); + + switchee_contref_last_ancestor.set_parent_stack_chain(env, builder, &handler_stack_chain); + + (switchee_contref_csi, switchee_contref_last_ancestor) + }; + + // Update VMContext/Store: Update active continuation and `VMRuntimeLimits`. + { + vmctx_set_active_continuation(env, builder, vmctx, switchee_contref.address); + + switchee_contref_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); + } + + // Perform actual stack switch + { + let switcher_last_ancestor_fs = + switcher_contref_last_ancestor.get_fiber_stack(env, builder); + let switcher_last_ancestor_cc = + switcher_last_ancestor_fs.load_control_context(env, builder); + + let switchee_last_ancestor_fs = + switchee_contref_last_ancestor.get_fiber_stack(env, builder); + let switchee_last_ancestor_cc = + switchee_last_ancestor_fs.load_control_context(env, builder); + + // The stack switch involves the following control contexts (e.g., IP, + // SP, FP, ...): + // - `switchee_last_ancestor_cc` contains the information to continue + // execution in the switchee/target continuation. + // - `switcher_last_ancestor_cc` contains the information about how to + // continue execution once we suspend/return to the stack with the + // switch handler. + // + // In total, the following needs to happen: + // 1. Load control context at `switchee_last_ancestor_cc` to perform + // stack switch. + // 2. Move control context at `switcher_last_ancestor_cc` over to + // `switchee_last_ancestor_cc`. + // 3. Upon actual switch, save current control context at + // `switcher_last_ancestor_cc`. + // + // We implement this as follows: + // 1. We copy `switchee_last_ancestor_cc` to a temporary area on the + // stack (`tmp_control_context`). + // 2. We copy `switcher_last_ancestor_cc` over to + // `switchee_last_ancestor_cc`. + // 3. We invoke the stack switch instruction such that it reads from the + // temporary area, and writes to `switcher_last_ancestor_cc`. + // + // Note that the temporary area is only accessed once by the + // `stack_switch` instruction emitted later in this block, meaning that we + // don't have to worry about its lifetime. + // + // NOTE(frank-emrich) The implementation below results in one stack slot + // being created per switch instruction, even though multiple switch + // instructions in the same function could safely re-use the same stack + // slot. Thus, we could implement logic for sharing the stack slot by + // adding an appropriate field to `FuncEnvironment`. + // + // NOTE(frank-emrich) We could avoid the copying to a temporary area by + // making `stack_switch` do all of the necessary moving itself. However, + // that would be a rather ad-hoc change to how the instruction uses the + // two pointers given to it. + + let slot_size = ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + u32::try_from(CONTROL_CONTEXT_SIZE).unwrap(), + u8::try_from(env.pointer_type().bytes()).unwrap(), + ); + let slot = builder.create_sized_stack_slot(slot_size); + let tmp_control_context = builder.ins().stack_addr(env.pointer_type(), slot, 0); + + let flags = MemFlags::trusted(); + let mut offset: i32 = 0; + while offset < i32::try_from(CONTROL_CONTEXT_SIZE).unwrap() { + // switchee_last_ancestor_cc -> tmp control context + let tmp1 = + builder + .ins() + .load(env.pointer_type(), flags, switchee_last_ancestor_cc, offset); + builder + .ins() + .store(flags, tmp1, tmp_control_context, offset); + + // switcher_last_ancestor_cc -> switchee_last_ancestor_cc + let tmp2 = + builder + .ins() + .load(env.pointer_type(), flags, switcher_last_ancestor_cc, offset); + builder + .ins() + .store(flags, tmp2, switchee_last_ancestor_cc, offset); + + offset += env.pointer_type().bytes() as i32; + } + + let switch_payload = ControlEffect::encode_switch(env, builder).to_u64(); + + emit_debug_println!( + env, + builder, + "[switch] about to execute stack_switch, store_control_context_ptr is {:p}, load_control_context_ptr {:p}, tmp_control_context is {:p}", + switcher_last_ancestor_cc, + switchee_last_ancestor_cc, + tmp_control_context + ); + + let result = builder.ins().stack_switch( + switcher_last_ancestor_cc, + tmp_control_context, + switch_payload, + ); + + emit_debug_println!( + env, + builder, + "[switch] continuing after stack_switch in frame with stack chain ({}, {:p}), result is {:p}", + handler_stack_chain.to_raw_parts()[0], + handler_stack_chain.to_raw_parts()[1], + result + ); + + if cfg!(debug_assertions) { + // The only way to switch back to this point is by using resume or switch instructions. + let result_control_effect = ControlEffect::from_u64(result); + let result_discriminant = result_control_effect.signal(env, builder); + let is_resume = builder.ins().icmp_imm( + IntCC::Equal, + result_discriminant, + CONTROL_EFFECT_RESUME_DISCRIMINANT as i64, + ); + let is_switch = builder.ins().icmp_imm( + IntCC::Equal, + result_discriminant, + CONTROL_EFFECT_SWITCH_DISCRIMINANT as i64, + ); + let is_switch_or_resume = builder.ins().bor(is_switch, is_resume); + emit_debug_assert!(env, builder, is_switch_or_resume); + } + } + + // After switching back to the original stack: Load return values, they are + // stored on the switcher continuation. + let return_values = { + if cfg!(debug_assertions) { + // The originally active continuation (before the switch) should be active again. + let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + // This has a debug assertion that also checks that the `active_stack_chain` is indeed a continuation. + let active_contref = active_stack_chain.unchecked_get_continuation(env, builder); + emit_debug_assert_eq!(env, builder, switcher_contref.address, active_contref); + } + + let payloads = switcher_contref.values(env, builder); + let return_values = payloads.load_data_entries(env, builder, return_types); + // We consume the values and discard the buffer (allocated on this stack) + payloads.clear(env, builder, true); + return_values + }; + + Ok(return_values) +} diff --git a/crates/cranelift/src/stack_switching/mod.rs b/crates/cranelift/src/stack_switching/mod.rs new file mode 100644 index 000000000000..653191a769fa --- /dev/null +++ b/crates/cranelift/src/stack_switching/mod.rs @@ -0,0 +1,3 @@ +mod control_effect; +pub(crate) mod fatpointer; +pub(crate) mod instructions; diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index c32f98a73a15..5f7245822c9b 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -94,7 +94,7 @@ use std::vec::Vec; use wasmparser::{FuncValidator, MemArg, Operator, WasmModuleResources}; use wasmtime_environ::{ wasm_unsupported, DataIndex, ElemIndex, FuncIndex, GlobalIndex, MemoryIndex, Signed, - TableIndex, TypeConvert, TypeIndex, Unsigned, WasmRefType, WasmResult, + TableIndex, TagIndex, TypeConvert, TypeIndex, Unsigned, WasmHeapType, WasmRefType, WasmResult, }; /// Given a `Reachability`, unwrap the inner `T` or, when unreachable, set @@ -2876,54 +2876,150 @@ pub fn translate_operator( // representation, so we don't actually need to do anything. } - Operator::ContNew { cont_type_index: _ } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + Operator::ContNew { cont_type_index } => { + let cont_type_index = TypeIndex::from_u32(*cont_type_index); + let arg_types = environ.continuation_arguments(cont_type_index).to_vec(); + let result_types = environ.continuation_returns(cont_type_index).to_vec(); + let r = state.pop1(); + let contobj = + environ.translate_cont_new(builder, state, r, &arg_types, &result_types)?; + state.push1(contobj); } Operator::ContBind { - argument_index: _, - result_index: _, + argument_index, + result_index, } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + let src_types = environ.continuation_arguments(TypeIndex::from_u32(*argument_index)); + let dst_arity = environ + .continuation_arguments(TypeIndex::from_u32(*result_index)) + .len(); + let arg_count = src_types.len() - dst_arity; + + let arg_types = &src_types[0..arg_count]; + for arg_type in arg_types { + // We can't bind GC objects using cont.bind at the moment: We + // don't have the necessary infrastructure to traverse the + // buffers used by cont.bind when looking for GC roots. Thus, + // this crude check ensures that these buffers can never contain + // GC roots to begin with. + if arg_type.is_vmgcref_type_and_not_i31() { + return Err(wasmtime_environ::WasmError::Unsupported( + "cont.bind does not support GC types at the moment".into(), + )); + } + } + + let (original_contobj, args) = state.peekn(arg_count + 1).split_last().unwrap(); + + let new_contobj = environ.translate_cont_bind(builder, *original_contobj, args); + + state.popn(arg_count + 1); + state.push1(new_contobj); } - Operator::Suspend { tag_index: _ } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + Operator::Suspend { tag_index } => { + let tag_index = TagIndex::from_u32(*tag_index); + let param_types = environ.tag_params(tag_index).to_vec(); + let return_types: Vec<_> = environ + .tag_returns(tag_index) + .iter() + .map(|ty| crate::value_type(environ.isa, *ty)) + .collect(); + + let params = state.peekn(param_types.len()); + let param_count = params.len(); + + let return_values = + environ.translate_suspend(builder, tag_index.as_u32(), params, &return_types); + + state.popn(param_count); + state.pushn(&return_values); } Operator::Resume { - cont_type_index: _, - resume_table: _, + cont_type_index, + resume_table, } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + // We translate the block indices in the resumetable to actual Blocks. + let mut resumetable = vec![]; + for handle in &resume_table.handlers { + match handle { + wasmparser::Handle::OnLabel { tag, label } => { + let i = state.control_stack.len() - 1 - (*label as usize); + let frame = &mut state.control_stack[i]; + // This is side-effecting! + frame.set_branched_to_exit(); + resumetable.push((*tag, Some(frame.br_destination()))); + } + wasmparser::Handle::OnSwitch { tag } => { + resumetable.push((*tag, None)); + } + } + } + + let cont_type_index = TypeIndex::from_u32(*cont_type_index); + let arity = environ.continuation_arguments(cont_type_index).len(); + let (contobj, call_args) = state.peekn(arity + 1).split_last().unwrap(); + + let cont_return_vals = environ.translate_resume( + builder, + cont_type_index.as_u32(), + *contobj, + call_args, + resumetable.as_slice(), + )?; + + state.popn(arity + 1); // arguments + continuation + state.pushn(&cont_return_vals); } Operator::ResumeThrow { cont_type_index: _, tag_index: _, resume_table: _, - } => { - // TODO(10248) This depends on exception handling - return Err(wasmtime_environ::WasmError::Unsupported( - "resume.throw instructions not supported, yet".to_string(), - )); - } + } => todo!("unimplemented stack switching instruction"), Operator::Switch { - cont_type_index: _, - tag_index: _, + cont_type_index, + tag_index, } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + // Arguments of the continuation we are going to switch to + let continuation_argument_types = environ + .continuation_arguments(TypeIndex::from_u32(*cont_type_index)) + .to_vec(); + // Arity includes the continuation argument + let arity = continuation_argument_types.len(); + let (contobj, switch_args) = state.peekn(arity).split_last().unwrap(); + + // Type of the continuation we are going to create by suspending the + // currently running stack + let current_continuation_type = continuation_argument_types.last().unwrap(); + let current_continuation_type = current_continuation_type.unwrap_ref_type(); + + // Argument types of current_continuation_type. These will in turn + // be the types of the arguments we receive when someone switches + // back to this switch instruction + let current_continuation_arg_types: Vec<_> = match current_continuation_type.heap_type { + WasmHeapType::ConcreteCont(index) => { + let mti = index + .as_module_type_index() + .expect("Only supporting module type indices on switch for now"); + + environ + .continuation_arguments(TypeIndex::from_u32(mti.as_u32())) + .iter() + .map(|ty| crate::value_type(environ.isa, *ty)) + .collect() + } + _ => panic!("Invalid type on switch"), + }; + + let switch_return_values = environ.translate_switch( + builder, + *tag_index, + *contobj, + switch_args, + ¤t_continuation_arg_types, + )?; + + state.popn(arity); + state.pushn(&switch_return_values) } Operator::GlobalAtomicGet { .. } From 21215681cfcb3c22877ef5061bd62b04ab8c6f7c Mon Sep 17 00:00:00 2001 From: Frank Emrich Date: Mon, 21 Apr 2025 22:24:00 +0100 Subject: [PATCH 5/6] [pr3] base --- crates/test-util/src/wasmtime_wast.rs | 2 +- crates/test-util/src/wast.rs | 8 + crates/wast/src/wast.rs | 32 +- tests/all/debug/lldb.rs | 90 + tests/all/debug/testsuite/stack_switch.wat | 32 + tests/all/main.rs | 2 + tests/all/stack_switching.rs | 1511 +++++++++++++++++ .../stack-switching/cont_args.wast | 105 ++ .../stack-switching/cont_bind1.wast | 31 + .../stack-switching/cont_bind2.wast | 32 + .../stack-switching/cont_bind3.wast | 41 + .../stack-switching/cont_bind4.wast | 104 ++ .../stack-switching/cont_bind_null.wast | 12 + .../stack-switching/cont_forwarding1.wast | 50 + .../stack-switching/cont_forwarding2.wast | 40 + .../stack-switching/cont_forwarding3.wast | 45 + .../stack-switching/cont_forwarding4.wast | 56 + .../stack-switching/cont_forwarding5.wast | 58 + .../stack-switching/cont_forwarding6.wast | 56 + .../stack-switching/cont_forwarding7.wast | 54 + .../stack-switching/cont_forwarding8.wast | 63 + .../stack-switching/cont_nary.wast | 90 + .../stack-switching/cont_nested1.wast | 53 + .../stack-switching/cont_nested2.wast | 65 + .../stack-switching/cont_nested3.wast | 41 + .../stack-switching/cont_nested4.wast | 45 + .../stack-switching/cont_nested5.wast | 69 + .../stack-switching/cont_nested6.wast | 45 + .../stack-switching/cont_new.wast | 17 + .../stack-switching/cont_new_null.wast | 12 + .../stack-switching/cont_resume.wast | 18 + .../stack-switching/cont_resume1.wast | 28 + .../stack-switching/cont_resume2.wast | 65 + .../stack-switching/cont_resume_null.wast | 11 + .../stack-switching/cont_resume_return.wast | 14 + .../stack-switching/cont_return.wast | 17 + .../stack-switching/cont_suspend.wast | 26 + .../stack-switching/cont_table.wast | 87 + .../stack-switching/cont_twice.wast | 16 + tests/misc_testsuite/stack-switching/dup.wast | 27 + .../stack-switching/linking_tags1.wast | 37 + .../stack-switching/linking_tags2.wast | 136 ++ .../stack-switching/switch1.wast | 35 + .../stack-switching/switch2.wast | 71 + .../stack-switching/switch3.wast | 95 ++ .../stack-switching/switch4.wast | 52 + .../stack-switching/switch5.wast | 54 + .../stack-switching/switch6.wast | 37 + .../stack-switching/switch7.wast | 52 + .../stack-switching/switch8.wast | 36 + .../stack-switching/switch_null.wast | 16 + .../stack-switching/unhandled.wast | 24 + 52 files changed, 3810 insertions(+), 5 deletions(-) create mode 100644 tests/all/debug/testsuite/stack_switch.wat create mode 100644 tests/all/stack_switching.rs create mode 100644 tests/misc_testsuite/stack-switching/cont_args.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_bind1.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_bind2.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_bind3.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_bind4.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_bind_null.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_forwarding1.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_forwarding2.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_forwarding3.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_forwarding4.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_forwarding5.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_forwarding6.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_forwarding7.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_forwarding8.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_nary.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_nested1.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_nested2.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_nested3.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_nested4.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_nested5.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_nested6.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_new.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_new_null.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_resume.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_resume1.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_resume2.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_resume_null.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_resume_return.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_return.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_suspend.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_table.wast create mode 100644 tests/misc_testsuite/stack-switching/cont_twice.wast create mode 100644 tests/misc_testsuite/stack-switching/dup.wast create mode 100644 tests/misc_testsuite/stack-switching/linking_tags1.wast create mode 100644 tests/misc_testsuite/stack-switching/linking_tags2.wast create mode 100644 tests/misc_testsuite/stack-switching/switch1.wast create mode 100644 tests/misc_testsuite/stack-switching/switch2.wast create mode 100644 tests/misc_testsuite/stack-switching/switch3.wast create mode 100644 tests/misc_testsuite/stack-switching/switch4.wast create mode 100644 tests/misc_testsuite/stack-switching/switch5.wast create mode 100644 tests/misc_testsuite/stack-switching/switch6.wast create mode 100644 tests/misc_testsuite/stack-switching/switch7.wast create mode 100644 tests/misc_testsuite/stack-switching/switch8.wast create mode 100644 tests/misc_testsuite/stack-switching/switch_null.wast create mode 100644 tests/misc_testsuite/stack-switching/unhandled.wast diff --git a/crates/test-util/src/wasmtime_wast.rs b/crates/test-util/src/wasmtime_wast.rs index bbbc161fb829..092f13c72739 100644 --- a/crates/test-util/src/wasmtime_wast.rs +++ b/crates/test-util/src/wasmtime_wast.rs @@ -74,7 +74,7 @@ pub fn apply_test_config(config: &mut Config, test_config: &wast::TestConfig) { // To avoid needing to enable all of them at once implicitly enable // downstream proposals once the end proposal is enabled (e.g. when enabling // gc that also enables function-references and reference-types). - let function_references = gc || function_references.unwrap_or(false); + let function_references = stack_switching || gc || function_references.unwrap_or(false); let reference_types = function_references || reference_types.unwrap_or(false); let simd = relaxed_simd || simd.unwrap_or(false); diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rs index 313c66118757..a5353efeadec 100644 --- a/crates/test-util/src/wast.rs +++ b/crates/test-util/src/wast.rs @@ -430,6 +430,14 @@ impl WastTest { } } + if cfg!(not(all(unix, target_arch = "x86_64"))) { + // Stack switching is not implemented on platforms other than x64 + // unix, the corresponding tests will fail. + if self.path.parent().unwrap().ends_with("stack-switching") { + return true; + } + } + if config.compiler.should_fail(&self.config) { return true; } diff --git a/crates/wast/src/wast.rs b/crates/wast/src/wast.rs index 7e871b9d2473..eed6d7760daf 100644 --- a/crates/wast/src/wast.rs +++ b/crates/wast/src/wast.rs @@ -439,6 +439,26 @@ where bail!("expected '{}', got '{}'", expected, actual) } + fn assert_suspension(&self, result: Outcome, expected: &str) -> Result<()> { + let trap = match result { + Outcome::Ok(values) => bail!("expected suspension, got {:?}", values), + Outcome::Trap(t) => t, + }; + let actual = format!("{trap:?}"); + if actual.contains(expected) + || actual.contains("unhandled tag") + || actual.contains("Calling suspend outside of a continuation") + { + Ok(()) + } else { + bail!( + "assert_suspension: expected '{}', got '{}'", + expected, + actual + ) + } + } + /// Run a wast script from a byte buffer. pub fn run_buffer(&mut self, filename: &str, wast: &[u8]) -> Result<()> { let wast = str::from_utf8(wast)?; @@ -607,6 +627,14 @@ where } } AssertException { .. } => bail!("unimplemented assert_exception"), + AssertSuspension { + span: _, + exec, + message, + } => { + let result = self.perform_execute(exec)?; + self.assert_suspension(result, message)?; + } Thread(thread) => { let mut core_linker = Linker::new(self.store.engine()); @@ -647,10 +675,6 @@ where .join() .unwrap()?; } - - AssertSuspension { .. } => { - bail!("unimplemented wast directive"); - } } Ok(()) diff --git a/tests/all/debug/lldb.rs b/tests/all/debug/lldb.rs index dbdc77c922e7..8e50a85f552c 100644 --- a/tests/all/debug/lldb.rs +++ b/tests/all/debug/lldb.rs @@ -551,3 +551,93 @@ check: exited with status = 0 Ok(()) } } + +#[cfg(all(feature = "stack-switching", unix, target_arch = "x86_64"))] +mod stack_switching { + use super::{check_lldb_output, lldb_with_script}; + use anyhow::Result; + + /// Checks that we get backtraces including the entire continuation chain when + /// using just FP walking. + #[test] + #[ignore] + pub fn debug_info_disabled() -> Result<()> { + let output = lldb_with_script( + &[ + "-Ccache=n", + "-Ddebug-info=n", + "-Wstack-switching", + "--invoke", + "entry", + "tests/all/debug/testsuite/stack_switch.wat", + ], + r#"r +bt +"#, + )?; + + // We are running without debug-info enabled, so we will not get Wasm + // function names in the output. Instead, we look for + // wasmtime_continuation_start, the trampoline at the bottom of all + // continuation stacks. + // + // This directive matches lines like this: + // frame #12: 0x0000555558f18fc9 wasmtime`wasmtime_continuation_start + 9 + let check = r#" + check: frame #$(=[0-9]+): 0x$(=[0-9a-f]+) wasmtime`wasmtime_continuation_start + "#; + + // Our stack_switch.wat file traps inside 3 levels of nested continuations. + // Thus, we must have 3 stack frames at function `wasmtime_continuation_start`. + check_lldb_output(&output, &check.repeat(3))?; + + Ok(()) + } + + /// Checks that we get backtraces including the entire continuation chain when + /// using just FP walking. + #[test] + #[ignore] + pub fn debug_info_enabled() -> Result<()> { + let output = lldb_with_script( + &[ + "-Ccache=n", + "-Ddebug-info=y", + "-Wstack-switching", + "--invoke", + "entry", + "tests/all/debug/testsuite/stack_switch.wat", + ], + r#"r +bt +"#, + )?; + + // We are running with debug-info enabled, so we get Wasm + // function names in the backtrace. + // + // Creates directive matching lines like this: + // frame #13: 0x00007ffff4e4a5be JIT(0x55555bc24b10)`c at .wasm:90 + // where the string is parameterised over the function name (c in the + // example above). + let check = |name: &str| { + format!( + "check: frame #$(=[0-9]+): 0x$(=[0-9a-f]+) JIT(0x$(=[0-9a-f]+))`{name} at .wasm" + ) + }; + + // Our stack_switch.wat file traps inside 3 levels of nested continuations. + // Let's check that our backtrace contains all the functions, even those in + // parent continuations. + check_lldb_output( + &output, + &["f", "e", "d", "c", "b", "a", "entry"] + .into_iter() + .map(check) + .collect::>() + .join("\n"), + )?; + + Ok(()) + } +} diff --git a/tests/all/debug/testsuite/stack_switch.wat b/tests/all/debug/testsuite/stack_switch.wat new file mode 100644 index 000000000000..58fc5a88177b --- /dev/null +++ b/tests/all/debug/testsuite/stack_switch.wat @@ -0,0 +1,32 @@ +(module + (type $ft (func)) + (type $ct (cont $ft)) + + (func $entry (export "entry") + (call $a) + ) + + (func $a (export "a") + (resume $ct (cont.new $ct (ref.func $b))) + ) + + (func $b (export "b") + (call $c) + ) + + (func $c (export "c") + (resume $ct (cont.new $ct (ref.func $d))) + ) + + (func $d (export "d") + (call $e) + ) + + (func $e (export "e") + (resume $ct (cont.new $ct (ref.func $f))) + ) + + (func $f (export "f") + (unreachable) + ) +) diff --git a/tests/all/main.rs b/tests/all/main.rs index e09a7abda0b3..79a977788e91 100644 --- a/tests/all/main.rs +++ b/tests/all/main.rs @@ -41,6 +41,8 @@ mod pulley; mod relocs; mod stack_creator; mod stack_overflow; +#[cfg(all(feature = "stack-switching", unix, target_arch = "x86_64"))] +mod stack_switching; mod store; mod structs; mod table; diff --git a/tests/all/stack_switching.rs b/tests/all/stack_switching.rs new file mode 100644 index 000000000000..d9ef5085d608 --- /dev/null +++ b/tests/all/stack_switching.rs @@ -0,0 +1,1511 @@ +use anyhow::Result; +use wasmtime::*; + +mod test_utils { + use anyhow::{bail, Result}; + use std::any::*; + use std::panic::AssertUnwindSafe; + use wasmtime::*; + + pub struct Runner { + pub engine: Engine, + pub store: Store<()>, + } + + impl Runner { + pub fn new() -> Runner { + let mut config = Config::default(); + config.wasm_function_references(true); + config.wasm_stack_switching(true); + // TODO(frank-emrich) Should this be implied by enabling + // stack-switching above? + config.wasm_exceptions(true); + // Required in order to use recursive types. + config.wasm_gc(true); + + let engine = Engine::new(&config).unwrap(); + + let store = Store::<()>::new(&engine, ()); + + Runner { engine, store } + } + + /// Uses this `Runner` to run the module defined in `wat`, satisfying + /// its imports using `imports`. The module must export a function + /// `entry`, taking no parameters and returning `Results`. + pub fn run_test( + mut self, + wat: &str, + imports: &[Extern], + ) -> Result { + let module = Module::new(&self.engine, wat)?; + + let instance = Instance::new(&mut self.store, &module, imports)?; + let entry = instance.get_typed_func::<(), Results>(&mut self.store, "entry")?; + + entry.call(&mut self.store, ()) + } + + /// Uses this `Runner` to run the module defined in `wat`, satisfying + /// its imports using `imports`. The module must export a function + /// `entry`, taking no parameters and without return values. Execution + /// of `entry` is expected to cause a panic (and that this is panic is + /// not handled by wasmtime previously). + /// Returns the `Error` payload. + pub fn run_test_expect_panic( + mut self, + wat: &str, + imports: &[Extern], + ) -> Box { + let module = Module::new(&self.engine, wat).unwrap(); + + let instance = Instance::new(&mut self.store, &module, imports).unwrap(); + let entry = instance.get_func(&mut self.store, "entry").unwrap(); + + std::panic::catch_unwind(AssertUnwindSafe(|| { + drop(entry.call(&mut self.store, &[], &mut [])) + })) + .unwrap_err() + } + } + + /// Creates a simple Host function that increments an i32 + pub fn make_i32_inc_host_func(runner: &mut Runner) -> Func { + Func::new( + &mut runner.store, + FuncType::new(&runner.engine, vec![ValType::I32], vec![ValType::I32]), + |mut _caller, args: &[Val], results: &mut [Val]| { + let res = match args { + [Val::I32(i)] => i + 1, + _ => bail!("Error: Received illegal argument (should be single i32)"), + }; + results[0] = Val::I32(res); + Ok(()) + }, + ) + } + + /// Creates a host function of type i32 -> i32. `export_func` must denote an + /// exported function of type i32 -> i32. The created host function + /// increments its argument by 1, passes it to the exported function, and in + /// turn increments the result before returning it as the overall result. + pub fn make_i32_inc_via_export_host_func( + runner: &mut Runner, + export_func: &'static str, + ) -> Func { + Func::new( + &mut runner.store, + FuncType::new(&runner.engine, vec![ValType::I32], vec![ValType::I32]), + |mut caller, args: &[Val], results: &mut [Val]| { + let export = caller + .get_export(export_func) + .ok_or(anyhow::anyhow!("could not get export"))?; + let func = export + .into_func() + .ok_or(anyhow::anyhow!("export is not a Func"))?; + let func_typed = func.typed::(caller.as_context())?; + let arg = args[0].unwrap_i32(); + let res = func_typed.call(caller.as_context_mut(), arg + 1)?; + results[0] = Val::I32(res + 1); + Ok(()) + }, + ) + } + + /// Creates a function without parameters or return values that simply calls + /// the given function. + pub fn make_call_export_host_func(runner: &mut Runner, export_func: &'static str) -> Func { + Func::new( + &mut runner.store, + FuncType::new(&runner.engine, vec![], vec![]), + |mut caller, _args: &[Val], _results: &mut [Val]| { + let export = caller + .get_export(export_func) + .ok_or(anyhow::anyhow!("could not get export"))?; + let func = export + .into_func() + .ok_or(anyhow::anyhow!("export is not a Func"))?; + let func_typed = func.typed::<(), ()>(caller.as_context())?; + let _res = func_typed.call(caller.as_context_mut(), ())?; + Ok(()) + }, + ) + } + + pub fn make_panicking_host_func(store: &mut Store<()>, msg: &'static str) -> Func { + Func::wrap(store, move || -> () { std::panic::panic_any(msg) }) + } +} + +mod wasi { + use anyhow::Result; + use wasmtime::{Config, Engine, Linker, Module, Store}; + use wasmtime_wasi::p2::WasiCtxBuilder; + use wasmtime_wasi::preview1::{self, WasiP1Ctx}; + + fn run_wasi_test(wat: &'static str) -> Result { + // Construct the wasm engine with async support disabled. + let mut config = Config::new(); + config + .async_support(false) + .wasm_function_references(true) + .wasm_stack_switching(true); + let engine = Engine::new(&config)?; + + // Add the WASI preview1 API to the linker (will be implemented in terms of + // the preview2 API) + let mut linker: Linker = Linker::new(&engine); + preview1::add_to_linker_sync(&mut linker, |t| t)?; + + // Add capabilities (e.g. filesystem access) to the WASI preview2 context here. + let wasi_ctx = WasiCtxBuilder::new().inherit_stdio().build_p1(); + + let mut store: Store = Store::new(&engine, wasi_ctx); + + // Instantiate our wasm module. + let module = Module::new(&engine, wat)?; + let func = linker + .module(&mut store, "", &module)? + .get_default(&mut store, "")? + .typed::<(), i32>(&store)?; + + // Invoke the WASI program default function. + func.call(&mut store, ()) + } + + async fn run_wasi_test_async(wat: &'static str) -> Result { + // Construct the wasm engine with async support enabled. + let mut config = Config::new(); + config + .async_support(true) + .wasm_function_references(true) + .wasm_stack_switching(true); + let engine = Engine::new(&config)?; + + // Add the WASI preview1 API to the linker (will be implemented in terms of + // the preview2 API) + let mut linker: Linker = Linker::new(&engine); + preview1::add_to_linker_async(&mut linker, |t| t)?; + + // Add capabilities (e.g. filesystem access) to the WASI preview2 context here. + let wasi_ctx = WasiCtxBuilder::new().inherit_stdio().build_p1(); + + let mut store: Store = Store::new(&engine, wasi_ctx); + + // Instantiate our wasm module. + let module = Module::new(&engine, wat)?; + let func = linker + .module_async(&mut store, "", &module) + .await? + .get_default(&mut store, "")? + .typed::<(), i32>(&store)?; + + // Invoke the WASI program default function. + func.call_async(&mut store, ()).await + } + + static WRITE_SOMETHING_WAT: &'static str = &r#" +(module + (type $ft (func (result i32))) + (type $ct (cont $ft)) + (import "wasi_snapshot_preview1" "fd_write" + (func $print (param $fd i32) + (param $iovec i32) + (param $len i32) + (param $written i32) (result i32))) + (memory 1) + (export "memory" (memory 0)) + + ;; 9 is the offset to write to + (data (i32.const 9) "something\n") + + (func $f (result i32) + (i32.const 0) ;; offset + (i32.const 9) ;; value start of the string + (i32.store) + + (i32.const 4) ;; offset + (i32.const 11) ;; value, the length of the string + (i32.store offset=0 align=2) ;; size_buf_len + + (i32.const 1) ;; 1 for stdout + (i32.const 0) ;; 0 as we stored the beginning of __wasi_ciovec_t + (i32.const 1) ;; + (i32.const 20) ;; nwritten + (call $print) + ) + (elem declare func $f) + + (func (export "_start") (result i32) + (ref.func $f) + (cont.new $ct) + (resume $ct) + ) +)"#; + + #[test] + #[cfg_attr(miri, ignore)] + fn write_something_test() -> Result<()> { + assert_eq!(run_wasi_test(WRITE_SOMETHING_WAT)?, 0); + Ok(()) + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn write_something_test_async() -> Result<()> { + assert_eq!(run_wasi_test_async(WRITE_SOMETHING_WAT).await?, 0); + Ok(()) + } + + static SCHED_YIELD_WAT: &'static str = r#" +(module + (type $ft (func (result i32))) + (type $ct (cont $ft)) + (import "wasi_snapshot_preview1" "sched_yield" + (func $sched_yield (result i32))) + (memory 1) + (export "memory" (memory 0)) + + (func $g (result i32) + (call $sched_yield)) + (elem declare func $g) + + (func (export "_start") (result i32) + (cont.new $ct (ref.func $g)) + (resume $ct) + ) +)"#; + + #[test] + #[cfg_attr(miri, ignore)] + fn sched_yield_test() -> Result<()> { + assert_eq!(run_wasi_test(SCHED_YIELD_WAT)?, 0); + Ok(()) + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn sched_yield_test_async() -> Result<()> { + assert_eq!(run_wasi_test_async(SCHED_YIELD_WAT).await?, 0); + Ok(()) + } +} + +/// Test that two distinct instantiations of the same module yield +/// different control tag identities. +#[test] +#[cfg_attr(miri, ignore)] +fn inter_instance_suspend() -> Result<()> { + let mut config = Config::default(); + config.wasm_function_references(true); + config.wasm_stack_switching(true); + // TODO(frank-emrich) Should this be implied by enabling stack-switching + // above? + config.wasm_exceptions(true); + + let engine = Engine::new(&config)?; + + let mut store = Store::<()>::new(&engine, ()); + + let wat_other = r#" + (module + + (type $ft (func)) + (type $ct (cont $ft)) + (tag $tag) + + + (func $suspend (export "suspend") + (suspend $tag) + ) + + (func $resume (export "resume") (param $f (ref $ct)) + (block $handler (result (ref $ct)) + (resume $ct (on $tag $handler) (local.get $f)) + (unreachable) + ) + (drop) + ) + ) + "#; + + let wat_main = r#" + (module + + (type $ft (func)) + (type $ct (cont $ft)) + + (import "other" "suspend" (func $suspend)) + (import "other" "resume" (func $resume (param (ref $ct)))) + + (elem declare func $suspend) + + + (func $entry (export "entry") + (call $resume (cont.new $ct (ref.func $suspend))) + ) + ) + "#; + + let module_other = Module::new(&engine, wat_other)?; + + let other_inst1 = Instance::new(&mut store, &module_other, &[])?; + let other_inst2 = Instance::new(&mut store, &module_other, &[])?; + + // Crucially, suspend and resume are from two separate instances + // of the same module. + let suspend = other_inst1.get_func(&mut store, "suspend").unwrap(); + let resume = other_inst2.get_func(&mut store, "resume").unwrap(); + + let module_main = Module::new(&engine, wat_main)?; + let main_instance = Instance::new(&mut store, &module_main, &[suspend.into(), resume.into()])?; + let entry_func = main_instance.get_func(&mut store, "entry").unwrap(); + + let result = entry_func.call(&mut store, &[], &mut []); + assert!(result.is_err()); + Ok(()) +} + +/// Tests interaction with host functions. Note that the interaction with host +/// functions and traps is covered by the module `traps` further down. +mod host { + use super::test_utils::*; + use wasmtime::*; + + #[test] + #[cfg_attr(miri, ignore)] + /// Tests calling a host function from within a wasm function running inside a continuation. + /// Call chain: + /// $entry -resume-> a -call-> host_func_a + fn call_host_from_continuation() -> Result<()> { + let wat = r#" + (module + (type $ft (func (result i32))) + (type $ct (cont $ft)) + + (import "" "" (func $host_func_a (param i32) (result i32))) + + (func $a (export "a") (result i32) + (call $host_func_a (i32.const 122)) + ) + (func $entry (export "entry") (result i32) + (resume $ct (cont.new $ct (ref.func $a))) + ) + ) + "#; + + let mut runner = Runner::new(); + + let host_func_a = make_i32_inc_host_func(&mut runner); + + let result = runner.run_test::(wat, &[host_func_a.into()]).unwrap(); + assert_eq!(result, 123); + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + /// We re-enter wasm from a host function and execute a continuation. + /// Call chain: + /// $entry -call-> $a -call-> $host_func_a -call-> $b -resume-> $c + fn re_enter_wasm_ok1() -> Result<()> { + let wat = r#" + (module + (type $ft (func (param i32) (result i32))) + (type $ct (cont $ft)) + + (import "" "" (func $host_func_a (param i32) (result i32))) + + + (func $a (export "a") (param $x i32) (result i32) + (call $host_func_a (local.get $x)) + ) + + (func $b (export "b") (param $x i32) (result i32) + (resume $ct (local.get $x) (cont.new $ct (ref.func $c))) + ) + + (func $c (export "c") (param $x i32) (result i32) + (return (i32.add (local.get $x) (i32.const 1))) + ) + + + (func $entry (export "entry") (result i32) + (call $a (i32.const 120)) + ) + ) + "#; + + let mut runner = Runner::new(); + + let host_func_a = make_i32_inc_via_export_host_func(&mut runner, "b"); + + let result = runner.run_test::(wat, &[host_func_a.into()]).unwrap(); + assert_eq!(result, 123); + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + /// Similar to `re_enter_wasm_ok2, but we run a continuation before the host call. + /// Call chain: + /// $entry -call-> $a -call-> $host_func_a -call-> $b -resume-> $c + fn re_enter_wasm_ok2() -> Result<()> { + let wat = r#" + (module + (type $ft (func (param i32) (result i32))) + (type $ct (cont $ft)) + + (import "" "" (func $host_func_a (param i32) (result i32))) + + + (func $a (export "a") (param $x i32) (result i32) + ;; Running continuation before calling into host is fine + (resume $ct (local.get $x) (cont.new $ct (ref.func $c))) + (drop) + + (call $host_func_a (local.get $x)) + ) + + (func $b (export "b") (param $x i32) (result i32) + (resume $ct (local.get $x) (cont.new $ct (ref.func $c))) + ) + + (func $c (export "c") (param $x i32) (result i32) + (return (i32.add (local.get $x) (i32.const 1))) + ) + + + (func $entry (export "entry") (result i32) + (call $a (i32.const 120)) + ) + ) + "#; + + let mut runner = Runner::new(); + + let host_func_a = make_i32_inc_via_export_host_func(&mut runner, "b"); + + let result = runner.run_test::(wat, &[host_func_a.into()]).unwrap(); + assert_eq!(result, 123); + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + /// We re-enter wasm from a host function while we were already on a continuation stack. + /// This is currently forbidden (see wasmfx/wasmfxtime#109), but may be + /// allowed in the future. + /// Call chain: + /// $entry -resume-> $a -call-> $host_func_a -call-> $b + fn re_enter_wasm_ok3() -> Result<()> { + let wat = r#" + (module + (type $ft (func (param i32) (result i32))) + (type $ct (cont $ft)) + + (import "" "" (func $host_func_a (param i32) (result i32))) + + + (func $a (export "a") (param $x i32) (result i32) + (call $host_func_a (local.get $x)) + ) + + + (func $b (export "b") (param $x i32) (result i32) + (return (i32.add (local.get $x) (i32.const 1))) + ) + + + (func $entry (export "entry") (result i32) + (resume $ct (i32.const 120) (cont.new $ct (ref.func $a))) + ) + ) + "#; + let mut runner = Runner::new(); + + let host_func_a = make_i32_inc_via_export_host_func(&mut runner, "b"); + + let result = runner.run_test::(wat, &[host_func_a.into()]).unwrap(); + assert_eq!(result, 123); + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + /// After crossing from the host back into wasm, we suspend to a tag that is + /// handled by the surrounding function (i.e., without needing to cross the + /// host frame to reach the handler). + /// Call chain: + /// $entry -resume-> $a -call-> $host_func_a -call-> $b -resume-> $c + fn call_host_from_continuation_nested_suspend_ok() -> Result<()> { + let wat = r#" + (module + (type $ft (func (param i32) (result i32))) + (type $ct (cont $ft)) + (tag $t (result i32)) + + (import "" "" (func $host_func_a (param i32) (result i32))) + + + (func $a (export "a") (param $x i32) (result i32) + (call $host_func_a (local.get $x)) + ) + + + (func $b (export "b") (param $x i32) (result i32) + (block $h (result (ref $ct)) + (resume $ct (on $t $h) (local.get $x) (cont.new $ct (ref.func $c))) + (unreachable) + ) + (drop) + ;; note that we do not run the continuation to completion + (i32.add (local.get $x) (i32.const 1)) + ) + + (func $c (export "c") (param $x i32) (result i32) + (suspend $t) + ) + + + (func $entry (export "entry") (result i32) + (resume $ct (i32.const 120) (cont.new $ct (ref.func $a))) + ) + ) + "#; + + let mut runner = Runner::new(); + + let host_func_a = make_i32_inc_via_export_host_func(&mut runner, "b"); + + let result = runner.run_test::(wat, &[host_func_a.into()]).unwrap(); + assert_eq!(result, 123); + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + /// Similar to `call_host_from_continuation_nested_suspend_ok`. However, we + /// suspend to a tag that is only handled if we were to cross a host + /// function boundary. That's not allowed, so we effectively suspend with an + /// unhandled tag. + /// Call chain: + /// $entry -resume-> $a -call-> $host_func_a -call-> $b -resume-> $c + fn call_host_from_continuation_nested_suspend_unhandled() -> Result<()> { + let wat = r#" + (module + (type $ft (func (param i32) (result i32))) + (type $ct (cont $ft)) + (tag $t (result i32)) + + (import "" "" (func $host_func_a (param i32) (result i32))) + + + (func $a (export "a") (param $x i32) (result i32) + (call $host_func_a (local.get $x)) + ) + + + (func $b (export "b") (param $x i32) (result i32) + (resume $ct (local.get $x) (cont.new $ct (ref.func $c))) + ) + + (func $c (export "c") (param $x i32) (result i32) + (suspend $t) + ) + + + (func $entry (export "entry") (result i32) + (block $h (result (ref $ct)) + (resume $ct + (on $t $h) + (i32.const 122) + (cont.new $ct (ref.func $a))) + (return) + ) + (unreachable) + ) + ) + "#; + + let mut runner = Runner::new(); + + let host_func_a = Func::new( + &mut runner.store, + FuncType::new(&runner.engine, vec![ValType::I32], vec![ValType::I32]), + |mut caller, args: &[Val], results: &mut [Val]| { + let export = caller + .get_export("b") + .ok_or(anyhow::anyhow!("could not get export"))?; + let func = export + .into_func() + .ok_or(anyhow::anyhow!("export is not a Func"))?; + + let func_typed = func.typed::(caller.as_context())?; + let arg = args[0].unwrap_i32(); + let res = func_typed.call(caller.as_context_mut(), arg + 1); + let err = res.unwrap_err(); + + assert!(err.root_cause().is::()); + assert_eq!(*err.downcast_ref::().unwrap(), Trap::UnhandledTag); + + let trace = err.downcast_ref::().unwrap(); + let frames: Vec<_> = trace + .frames() + .iter() + .map(|frame| { + frame + .func_name() + .expect("Expecting all functions in actual backtrace to have names") + }) + .rev() + .collect(); + assert_eq!(frames, &["entry", "a", "b", "c"]); + + results[0] = Val::I32(arg + 1); + Ok(()) + }, + ); + + let result = runner.run_test::(wat, &[host_func_a.into()]).unwrap(); + assert_eq!(result, 123); + Ok(()) + } +} + +mod traps { + use super::test_utils::*; + use wasmtime::*; + + fn backtrace_from_err(err: &Error) -> impl Iterator { + let trace = err.downcast_ref::().unwrap(); + + trace + .frames() + .iter() + .map(|frame| { + frame + .func_name() + .expect("Expecting all functions in actual backtrace to have names") + }) + .rev() + } + + /// Runs the module given as `wat`. We expect execution to cause the + /// `expected_trap` and a backtrace containing exactly the function names + /// given by `expected_backtrace`. + fn run_test_expect_trap_backtrace(wat: &str, expected_trap: Trap, expected_backtrace: &[&str]) { + let runner = Runner::new(); + let result = runner.run_test::<()>(wat, &[]); + + let err = result.expect_err("Was expecting wasm execution to yield error"); + + assert!(err.root_cause().is::()); + assert_eq!(*err.downcast_ref::().unwrap(), expected_trap); + + let actual_func_name_it = backtrace_from_err(&err); + + let expected_func_name_it = expected_backtrace.iter().copied(); + assert!(actual_func_name_it.eq(expected_func_name_it)); + } + + #[test] + #[cfg_attr(miri, ignore)] + /// Tests that we get correct backtraces if we trap deep inside multiple continuations. + /// Call chain: + /// $entry -call-> $a -resume-> $b -call-> $c -resume-> $d -call-> $e -resume-> $f + /// Here, $f traps. + fn trap_in_continuation_nested() -> Result<()> { + let wat = r#" + (module + (type $ft (func)) + (type $ct (cont $ft)) + + (func $entry (export "entry") + (call $a) + ) + + (func $a (export "a") + (resume $ct (cont.new $ct (ref.func $b))) + ) + + (func $b (export "b") + (call $c) + ) + + (func $c (export "c") + (resume $ct (cont.new $ct (ref.func $d))) + ) + + (func $d (export "d") + (call $e) + ) + + (func $e (export "e") + (resume $ct (cont.new $ct (ref.func $f))) + ) + + (func $f (export "f") + (unreachable) + ) + ) + "#; + + run_test_expect_trap_backtrace( + wat, + Trap::UnreachableCodeReached, + &["entry", "a", "b", "c", "d", "e", "f"], + ); + + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + /// Tests that we get correct backtraces if we trap after returning from one + /// continuation to its parent. + fn trap_in_continuation_back_to_parent() -> Result<()> { + let wat = r#" + (module + (type $ft (func)) + (type $ct (cont $ft)) + + (func $entry (export "entry") + (call $a) + ) + + (func $a (export "a") + (resume $ct (cont.new $ct (ref.func $b))) + ) + + (func $b (export "b") + (call $c) + ) + + (func $c (export "c") + (resume $ct (cont.new $ct (ref.func $d))) + (unreachable) + ) + + (func $d (export "d") + (call $e) + ) + + (func $e (export "e")) + + ) + "#; + + run_test_expect_trap_backtrace( + wat, + Trap::UnreachableCodeReached, + &["entry", "a", "b", "c"], + ); + + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + /// Tests that we get correct backtraces if we trap after returning from + /// several continuations back to the main stack. + fn trap_in_continuation_back_to_main() -> Result<()> { + let wat = r#" + (module + (type $ft (func)) + (type $ct (cont $ft)) + + (func $entry (export "entry") + (call $a) + ) + + (func $a (export "a") + (resume $ct (cont.new $ct (ref.func $b))) + (unreachable) + ) + + (func $b (export "b") + (call $c) + ) + + (func $c (export "c") + (resume $ct (cont.new $ct (ref.func $d))) + ) + + (func $d (export "d") + (call $e) + ) + + (func $e (export "e")) + + ) + "#; + + run_test_expect_trap_backtrace(wat, Trap::UnreachableCodeReached, &["entry", "a"]); + + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + /// Tests that we get correct backtraces after suspending a continuation. + fn trap_in_continuation_suspend() -> Result<()> { + let wat = r#" + (module + (type $ft (func)) + (type $ct (cont $ft)) + + (tag $t) + + (func $entry (export "entry") + (call $a) + ) + + (func $a (export "a") + (resume $ct (cont.new $ct (ref.func $b))) + (unreachable) + ) + + (func $b (export "b") + (call $c) + ) + + (func $c (export "c") + (block $handler (result (ref $ct)) + (resume $ct (on $t $handler) (cont.new $ct (ref.func $d))) + (return) + ) + (unreachable) + ) + + (func $d (export "d") + (call $e) + ) + + (func $e (export "e") + (suspend $t) + ) + + ) + "#; + + run_test_expect_trap_backtrace( + wat, + Trap::UnreachableCodeReached, + &["entry", "a", "b", "c"], + ); + + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + /// Tests that we get correct backtraces after suspending a continuation and + /// then resuming it from a different stack frame. + fn trap_in_continuation_suspend_resume() -> Result<()> { + let wat = r#" + (module + (type $ft (func)) + (type $ct (cont $ft)) + + (tag $t) + + (func $entry (export "entry") + (call $a) + ) + + (func $a (export "a") + (resume $ct (cont.new $ct (ref.func $b))) + ) + + (func $b (export "b") + (resume $ct (call $c)) + ) + + (func $c (export "c") (result (ref $ct)) + (block $handler (result (ref $ct)) + (resume $ct (on $t $handler) (cont.new $ct (ref.func $d))) + + ;; We never want to get here, but also don't want to use + ;; (unreachable), which is the trap we deliberately use in + ;; this test. Instead, we call a null function ref here, + ;; which is guaranteed to trap. + (call_ref $ft (ref.null $ft)) + + (return (cont.new $ct (ref.func $d))) + ) + ;; implicitly returning the continuation here + ) + + (func $d (export "d") + (call $e) + (unreachable) + ) + + (func $e (export "e") + (suspend $t) + ) + + ) + "#; + + // Note that c does not appear in the stack trace: + // In $b, we resume the suspended computation, which started in $d, + // suspended in $e, and traps in $d + run_test_expect_trap_backtrace( + wat, + Trap::UnreachableCodeReached, + &["entry", "a", "b", "d"], + ); + + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + /// Tests that we get correct backtraces after suspending a continuation + /// where we need to forward to an outer handler. + fn trap_in_continuation_forward() -> Result<()> { + let wat = r#" + (module + (type $ft (func)) + (type $ct (cont $ft)) + (tag $t) + + (func $entry (export "entry") + (call $a) + ) + + (func $a (export "a") + (block $handler (result (ref $ct)) + (resume $ct (on $t $handler) (cont.new $ct (ref.func $b))) + ;; We don't actually want to get here + (return) + ) + (unreachable) + ) + + (func $b (export "b") + (call $c) + ) + + (func $c (export "c") + (resume $ct (cont.new $ct (ref.func $d))) + ) + + (func $d (export "d") + (call $e) + ) + + (func $e (export "e") + (suspend $t) + ) + + ) + "#; + + // Note that c does not appear in the stack trace: + // In $b, we resume the suspended computation, which started in $d, + // suspended in $e, and traps in $d + run_test_expect_trap_backtrace(wat, Trap::UnreachableCodeReached, &["entry", "a"]); + + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + /// Tests that we get correct backtraces after suspending a continuation + /// where we need to forward to an outer handler. We then resume the + /// continuation from within another continuation. + fn trap_in_continuation_forward_resume() -> Result<()> { + let wat = r#" + (module + (type $ft (func)) + (type $ct (cont $ft)) + (tag $t) + + (global $k (mut (ref null $ct)) (ref.null $ct)) + + (func $entry (export "entry") + (call $a) + ) + + (func $a (export "a") + (resume $ct (cont.new $ct (ref.func $b))) + ) + + (func $b (export "b") + (block $handler (result (ref $ct)) + (resume $ct (on $t $handler) (cont.new $ct (ref.func $c))) + ;; We don't actually want to get here + (return) + ) + (global.set $k) + ;; $f will resume $k + (resume $ct (cont.new $ct (ref.func $f))) + ) + + (func $c (export "c") + (resume $ct (cont.new $ct (ref.func $d))) + ) + + (func $d (export "d") + (call $e) + ) + + (func $e (export "e") + (suspend $t) + (unreachable) + ) + + (func $f (export "f") + (resume $ct (global.get $k)) + ) + ) + "#; + + run_test_expect_trap_backtrace( + wat, + Trap::UnreachableCodeReached, + &["entry", "a", "b", "f", "c", "d", "e"], + ); + + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + /// Tests that we get correct backtraces after switch. + /// We first create the a stack with the following shape: + /// entry -> a -> b, then switch, leading to + /// entry -> c -> d, at which point we resume the a -> b continuation: + /// entry -> c -> d -> a -> b + /// We trap at that point. + fn trap_switch_and_resume() -> Result<()> { + let wat = r#" + (module + (rec + (type $ft0 (func (param (ref null $ct0)))) + (type $ct0 (cont $ft0))) + + (type $ft1 (func)) + (type $ct1 (cont $ft1)) + + (tag $t) + + (func $a (type $ft1) + (cont.new $ct1 (ref.func $b)) + (resume $ct1) + ) + (elem declare func $a) + + (func $b (type $ft1) + (cont.new $ct0 (ref.func $c)) + (switch $ct0 $t) + + ;; we want a backtrace here + (unreachable) + ) + (elem declare func $b) + + (func $c (type $ft0) + (local.get 0) + (cont.new $ct0 (ref.func $d)) + (resume $ct0) + ) + (elem declare func $c) + + (func $d (type $ft0) + (block $handler (result (ref $ct1)) + (ref.null $ct0) ;; passed as payload + (local.get 0) ;; resumed + (resume $ct0 (on $t $handler)) + (unreachable) ;; f1 will suspend after the switch + ) + (resume $ct1) + ) + (elem declare func $d) + + (func $entry (export "entry") + (cont.new $ct1 (ref.func $a)) + (resume $ct1 (on $t switch)) + ) + ) + "#; + + run_test_expect_trap_backtrace( + wat, + Trap::UnreachableCodeReached, + &["entry", "c", "d", "a", "b"], + ); + + Ok(()) + } + + // Test that we get correct backtraces after trapping inside a continuation + // after re-entering Wasm while already inside a different continuation. + #[test] + #[cfg_attr(miri, ignore)] + fn trap_after_re_enter() -> Result<()> { + let wat = r#" + (module + (type $ft (func)) + (type $ct (cont $ft)) + (tag $t) + + (import "" "" (func $host_func_a)) + (import "" "" (func $host_func_b)) + + (func $entry (export "entry") + (call $a) + ) + + (func $a (export "a") + (resume $ct (cont.new $ct (ref.func $b))) + ) + + (func $b (export "b") + (call $host_func_a) + ) + + (func $c (export "c") + (resume $ct (cont.new $ct (ref.func $d))) + + ) + + (func $d (export "d") + (i32.const 0) + (call $host_func_b) + (drop) + ) + + (func $e (export "e") + (resume $ct (cont.new $ct (ref.func $f))) + ) + + (func $f (export "f") + (unreachable) + ) + ) + "#; + + let mut runner = Runner::new(); + let host_func_a = make_call_export_host_func(&mut runner, "c"); + let host_func_b = make_call_export_host_func(&mut runner, "e"); + + let result = runner.run_test::<()>(wat, &[host_func_a.into(), host_func_b.into()]); + let err = result.unwrap_err(); + + assert!(err.root_cause().is::()); + assert_eq!( + *err.downcast_ref::().unwrap(), + Trap::UnreachableCodeReached + ); + + let backtrace = backtrace_from_err(&err); + assert!(backtrace.eq(["entry", "a", "b", "c", "d", "e", "f"].into_iter())); + + Ok(()) + } + + // Tests that we properly clean up the instance/store after trapping while + // running inside a continuation: There must be no leftovers of the old + // stack chain if we re-use the instance later. + #[test] + #[cfg_attr(miri, ignore)] + fn reuse_instance_after_trap1() -> Result<()> { + let wat = r#" + (module + (type $ft (func)) + (type $ct (cont $ft)) + + (tag $t) + + (func $entry1 (export "entry1") + (local $c (ref $ct)) + (local.set $c (cont.new $ct (ref.func $a))) + (block $handlet (result (ref $ct)) + (resume $ct (on $t $handlet) (local.get $c)) + (return) + ) + (unreachable) + ) + + (func $a (export "a") + (unreachable) + ) + + (func $entry2 (export "entry2") + (suspend $t) + ) + ) + "#; + + let mut config = Config::default(); + config.wasm_function_references(true); + config.wasm_stack_switching(true); + // TODO(frank-emrich) Should this be implied by enabling stack-switching + // above? + config.wasm_exceptions(true); + + let engine = Engine::new(&config).unwrap(); + let mut store = Store::<()>::new(&engine, ()); + let module = Module::new(&engine, wat)?; + + let instance = Instance::new(&mut store, &module, &[])?; + + // We execute entry 1, which traps with (unreachable) while $a is running inside a continuation. + let entry1 = instance.get_typed_func::<(), ()>(&mut store, "entry1")?; + let result1 = entry1.call(&mut store, ()); + let err1 = result1.expect_err("Was expecting wasm execution to yield error"); + assert!(err1.root_cause().is::()); + assert_eq!( + *err1.downcast_ref::().unwrap(), + Trap::UnreachableCodeReached + ); + let trace1 = backtrace_from_err(&err1); + assert!(trace1.eq(["entry1", "a"].into_iter())); + + // Now we re-enter the instance and immediately suspend with tag $t. + // This should trap, as there is no handler for it. + // In particular, we must not try to use the handler for $t installed by $entry1. + let entry2 = instance.get_typed_func::<(), ()>(&mut store, "entry2")?; + let result2 = entry2.call(&mut store, ()); + let err2 = result2.unwrap_err(); + assert!(err2.root_cause().is::()); + assert_eq!(*err2.downcast_ref::().unwrap(), Trap::UnhandledTag); + let trace2 = backtrace_from_err(&err2); + assert!(trace2.eq(["entry2"].into_iter())); + + Ok(()) + } + + // Tests that we properly clean up the instance/store after trapping while + // running inside a continuation: + // This test is similar to `reuse_instance_after_trap1`, but here we don't + // trap the second time we enter the instance. + #[test] + #[cfg_attr(miri, ignore)] + fn reuse_instance_after_trap2() -> Result<()> { + let wat = r#" + (module + (type $ft (func)) + (type $ct (cont $ft)) + + (tag $t) + + (func $entry1 (export "entry1") + (local $c (ref $ct)) + (local.set $c (cont.new $ct (ref.func $a))) + (block $handlet (result (ref $ct)) + (resume $ct (on $t $handlet) (local.get $c)) + (return) + ) + (unreachable) + ) + + (func $entry2 (export "entry2") (param $x i32) (result i32) + (local $c (ref $ct)) + (local.set $c (cont.new $ct (ref.func $b))) + (block $handlet (result (ref $ct)) + (resume $ct (on $t $handlet) (local.get $c)) + (unreachable) + ) + ;; note that we don't run the continuation to completion. + (drop) + (i32.add (local.get $x) (i32.const 1)) + ) + + (func $a (export "a") + (unreachable) + ) + + (func $b (export "b") + (suspend $t) + ) + + ) + "#; + + let mut config = Config::default(); + config.wasm_function_references(true); + config.wasm_stack_switching(true); + // TODO(frank-emrich) Should this be implied by enabling stack-switching + // above? + config.wasm_exceptions(true); + + let engine = Engine::new(&config).unwrap(); + let mut store = Store::<()>::new(&engine, ()); + let module = Module::new(&engine, wat)?; + + let instance = Instance::new(&mut store, &module, &[])?; + + // We execute entry 1, which traps with (unreachable) while $a is running inside a continuation. + let entry1 = instance.get_typed_func::<(), ()>(&mut store, "entry1")?; + let result1 = entry1.call(&mut store, ()); + let err1 = result1.expect_err("Was expecting wasm execution to yield error"); + assert!(err1.root_cause().is::()); + assert_eq!( + *err1.downcast_ref::().unwrap(), + Trap::UnreachableCodeReached + ); + let trace1 = backtrace_from_err(&err1); + assert!(trace1.eq(["entry1", "a"].into_iter())); + + // Now we re-enter the instance and succesfully run some stack-switchy code. + let entry1 = instance.get_typed_func::(&mut store, "entry2")?; + let result1 = entry1.call(&mut store, 122); + let result_value = result1.unwrap(); + assert_eq!(result_value, 123); + + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + /// Tests that we get correct panic payloads if we panic deep inside multiple + /// continuations. Note that wasmtime does not create its own backtraces for panics. + fn panic_in_continuation() -> Result<()> { + let wat = r#" + (module + (type $ft (func)) + (type $ct (cont $ft)) + + (import "" "" (func $f)) + + (func $entry (export "entry") + (call $a) + ) + + (func $a (export "a") + (resume $ct (cont.new $ct (ref.func $b))) + ) + + (func $b (export "b") + (call $c) + ) + + (func $c (export "c") + (resume $ct (cont.new $ct (ref.func $d))) + ) + + (func $d (export "d") + (call $e) + ) + + (func $e (export "e") + (call $f) + ) + + ) + "#; + + let mut runner = Runner::new(); + + let msg = "Host function f panics"; + + let f = make_panicking_host_func(&mut runner.store, msg); + let error = runner.run_test_expect_panic(wat, &[f.into()]); + assert_eq!(error.downcast_ref::<&'static str>(), Some(&msg)); + + Ok(()) + } + + #[test] + #[cfg_attr(miri, ignore)] + fn stack_overflow_in_continuation() -> Result<()> { + let wat = r#" + (module + (type $ft (func (param i32))) + (type $ct (cont $ft)) + + (func $entry (export "entry") + (call $a) + ) + + (func $a (export "a") + ;; We ask for a billion recursive calls + (i32.const 1_000_000_000) + + (resume $ct (cont.new $ct (ref.func $overflow))) + ) + + (func $overflow (export "overflow") (param $i i32) + (block $continue + (local.get $i) + ;; return if $i == 0 + (br_if $continue) + (return) + ) + (i32.sub (local.get $i) (i32.const 1)) + (call $overflow) + ) + + ) + "#; + + let runner = Runner::new(); + + let error = runner + .run_test::<()>(wat, &[]) + .expect_err("Expecting execution to yield error"); + + assert!(error.root_cause().is::()); + assert_eq!(*error.downcast_ref::().unwrap(), Trap::StackOverflow); + + Ok(()) + } +} + +mod misc { + use super::test_utils::*; + use wasmtime::*; + + #[ignore] + #[test] + pub fn continuation_revision_counter_wraparound() -> Result<()> { + let wat = r#" +(module + (type $ft (func)) + (type $ct (cont $ft)) + + (tag $yield) + + (func $loop + (loop $loop + (suspend $yield) + (br $loop) + ) + ) + (elem declare func $loop) + + ;; Loops 65536 times to overflow the 16 bit revision counter on the continuation reference. + (func (export "entry") + (local $k (ref $ct)) + (local $i i32) + (local.set $k (cont.new $ct (ref.func $loop))) + (loop $go-again + (block $on-yield (result (ref $ct)) + (resume $ct (on $yield $on-yield) (local.get $k)) + (unreachable) + ) + (local.set $k) + (local.set $i (i32.add (i32.const 1) (local.get $i))) + (br_if $go-again (i32.lt_u (local.get $i) (i32.const 65536))) + ) + ) +) +"#; + + let runner = Runner::new(); + let error = runner + .run_test::<()>(wat, &[]) + .expect_err("expected an overflow"); + assert!(error.root_cause().is::()); + assert_eq!( + *error.downcast_ref::().unwrap(), + Trap::IntegerOverflow + ); + Ok(()) + } +} diff --git a/tests/misc_testsuite/stack-switching/cont_args.wast b/tests/misc_testsuite/stack-switching/cont_args.wast new file mode 100644 index 000000000000..8dba487cccec --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_args.wast @@ -0,0 +1,105 @@ +;;! stack_switching = true +;; This file tests passing arguments to functions used has continuations and +;; returning values from such continuations on ordinary (i.e., non-suspend) exit + +(module + + (type $unit_to_unit (func)) + (type $unit_to_int (func (result i32))) + (type $int_to_unit (func (param i32))) + (type $int_to_int (func (param i32) (result i32))) + + + (type $f1_t (func (param i32) (result i32))) + (type $f1_ct (cont $f1_t)) + + (type $f2_t (func (param i32) (result i32))) + (type $f2_ct (cont $f2_t)) + + (type $f3_t (func (param i32) (result i32))) + (type $f3_ct (cont $f3_t)) + + (type $res_unit_to_unit (cont $unit_to_unit)) + (type $res_int_to_unit (cont $int_to_unit)) + (type $res_int_to_int (cont $int_to_int)) + (type $res_unit_to_int (cont $unit_to_int)) + + (tag $e1_unit_to_unit) + (tag $e2_int_to_unit (param i32)) + (tag $e3_int_to_int (param i32) (result i32)) + + (global $i (mut i32) (i32.const 0)) + + + ;; Used for testing the passing of arguments to continuation function and returning values out of them + (func $f1 (export "f1") (param $x i32) (result i32) + (global.set $i (i32.add (global.get $i) (local.get $x))) + (suspend $e1_unit_to_unit) + (i32.add (i32.const 2) (local.get $x))) + + ;; Used for testing case where no suspend happens at all + (func $f2 (export "f2") (param $x i32) (result i32) + (global.set $i (i32.add (global.get $i) (local.get $x))) + (i32.add (i32.const 2) (local.get $x))) + + ;; Same as $f1, but additionally passes payloads to and from handler + (func $f3 (export "f3") (param $x i32) (result i32) + (i32.add (local.get $x) (i32.const 1)) + (suspend $e3_int_to_int) + ;; return x + value returned received back from $e3 + (i32.add (local.get $x))) + + + (func $test_case_1 (export "test_case_1") (result i32) + ;; remove this eventually + (global.set $i (i32.const 0)) + (block $on_e1 (result (ref $res_unit_to_int)) + (resume $f1_ct (on $e1_unit_to_unit $on_e1) (i32.const 100) (cont.new $f1_ct (ref.func $f1))) + ;; unreachable: we never intend to invoke the resumption when handling + ;; $e1 invoked from $f2 + (unreachable)) + ;; after on_e1, stack: [resumption] + (drop) ;; drop resumption + (global.get $i)) + + (func $test_case_2 (export "test_case_2") (result i32) + ;; remove this eventually + (global.set $i (i32.const 0)) + ;;(local $finish_f3 (ref $res_unit_to_unit)) + (block $on_e1 (result (ref $res_unit_to_int)) + (resume $f1_ct (on $e1_unit_to_unit $on_e1) (i32.const 49) (cont.new $f1_ct (ref.func $f1))) + (unreachable)) + ;; after on_e1, stack: [resumption] + ;;(local.set $finish_f2) + (resume $res_unit_to_int) + ;; the resume above resumes execution of f2, which finishes without further suspends + (i32.add (global.get $i))) + + (func $test_case_3 (export "test_case_3") (result i32) + ;; remove this eventually + (global.set $i (i32.const 0)) + (resume $f2_ct (i32.const 49) (cont.new $f2_ct (ref.func $f2))) + (i32.add (global.get $i))) + + + (func $test_case_4 (export "test_case_4") (result i32) + (local $k (ref $res_int_to_int)) + + (block $on_e3 (result i32 (ref $res_int_to_int)) + (resume $f3_ct (on $e3_int_to_int $on_e3) (i32.const 49) (cont.new $f3_ct (ref.func $f3))) + (unreachable)) + ;; after on_e3, expected stack: [50 resumption] + (local.set $k) + + ;; add 1 to value 50 received from f6 via tag e3, thus passing 51 back to it + (i32.add (i32.const 1)) + (resume $res_int_to_int (local.get $k)) + ;; expecting to get 49 (original argument to function) + 51 (passed above) back + ) + +) + +(assert_return (invoke "test_case_1") (i32.const 100)) +(assert_return (invoke "test_case_2") (i32.const 100)) +(assert_return (invoke "test_case_3") (i32.const 100)) +(assert_return (invoke "test_case_4") (i32.const 100)) diff --git a/tests/misc_testsuite/stack-switching/cont_bind1.wast b/tests/misc_testsuite/stack-switching/cont_bind1.wast new file mode 100644 index 000000000000..3c0460abc005 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_bind1.wast @@ -0,0 +1,31 @@ +;;! stack_switching = true +;; Simple test for cont.bind: cont.bind supplies 0 arguments + +(module + (type $unit_to_int (func (result i32))) + (type $int_int_to_int (func (param i32 i32) (result i32))) + (type $int_to_int (func (param i32) (result i32))) + + (type $ct0 (cont $unit_to_int)) + (type $ct1 (cont $int_int_to_int)) + + (tag $e) + + (func $g (param $x i32) (param $y i32) (result i32) + (suspend $e) + (i32.add (local.get $x) (local.get $y))) + (elem declare func $g) + + (func $test (export "test") (result i32) + (block $on_e (result (ref $ct0)) + (i32.const 49) ;; consumed by resume + (i32.const 51) ;; consumed by resume + (cont.new $ct1 (ref.func $g)) + (cont.bind $ct1 $ct1) + (resume $ct1 (on $e $on_e)) + (unreachable)) + ;; on_e + (resume $ct0)) +) + +(assert_return (invoke "test") (i32.const 100)) diff --git a/tests/misc_testsuite/stack-switching/cont_bind2.wast b/tests/misc_testsuite/stack-switching/cont_bind2.wast new file mode 100644 index 000000000000..e143239fafde --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_bind2.wast @@ -0,0 +1,32 @@ +;;! stack_switching = true +;; Simple test for cont.bind: cont.bind turns 2-arg continution into 1-arg one before calling resume + +(module + (type $unit_to_int (func (result i32))) + (type $int_int_to_int (func (param i32 i32) (result i32))) + (type $int_to_int (func (param i32) (result i32))) + + (type $ct0 (cont $unit_to_int)) + (type $ct1 (cont $int_to_int)) + (type $ct2 (cont $int_int_to_int)) + + (tag $e) + + (func $g (param $x i32) (param $y i32) (result i32) + (suspend $e) + (i32.add (local.get $x) (local.get $y))) + (elem declare func $g) + + (func $test (export "test") (result i32) + (block $on_e (result (ref $ct0)) + (i32.const 49) ;; consumed by resume + (i32.const 51) ;; consumed by cont.bind + (cont.new $ct2 (ref.func $g)) + (cont.bind $ct2 $ct1) + (resume $ct1 (on $e $on_e)) + (unreachable)) + ;; on_e + (resume $ct0)) +) + +(assert_return (invoke "test") (i32.const 100)) diff --git a/tests/misc_testsuite/stack-switching/cont_bind3.wast b/tests/misc_testsuite/stack-switching/cont_bind3.wast new file mode 100644 index 000000000000..e9a5e4a66257 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_bind3.wast @@ -0,0 +1,41 @@ +;;! stack_switching = true +;; Testing cont.bind on continuations received from suspending rather than cont.new. + +(module + (type $unit_to_int (func (result i32))) + (type $int_to_unit (func (param i32))) + (type $int_to_int (func (param i32) (result i32))) + (type $2int_to_int (func (param i32 i32) (result i32))) + (type $3int_to_int (func (param i32 i32 i32) (result i32))) + + (type $ct0 (cont $unit_to_int)) + (type $ct1 (cont $3int_to_int)) + (type $ct2 (cont $2int_to_int)) + (type $ct3 (cont $int_to_int)) + + (tag $e (param i32 i32) (result i32 i32 i32)) + + (func $g (result i32) + (suspend $e (i32.const 5) (i32.const 15)) + (i32.add) + (i32.add)) + (elem declare func $g) + + (func $test (export "test") (result i32) + (local $k (ref $ct1)) + (i32.const 35) ;; to be consumed by second call to cont.resume + (i32.const 45) ;; to be consumed by second call to cont.bind + (block $on_e (result i32 i32 (ref $ct1)) + (resume $ct0 (on $e $on_e) (cont.new $ct0 (ref.func $g))) + (unreachable)) + ;; on_e: + (local.set $k) + (i32.add) ;; add two values received from $e, leave on stack to be consumed by first call to cont.bind + (local.get $k) + (cont.bind $ct1 $ct2) ;; consumes the result (= 20) of the addition two lines earlier + (cont.bind $ct2 $ct3) ;; consumes the constant value 45 put on stack earlier + (resume $ct3) ;; consumes the constant value 35 put on stack earlier + ) +) + +(assert_return (invoke "test") (i32.const 100)) diff --git a/tests/misc_testsuite/stack-switching/cont_bind4.wast b/tests/misc_testsuite/stack-switching/cont_bind4.wast new file mode 100644 index 000000000000..a067dfabd832 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_bind4.wast @@ -0,0 +1,104 @@ +;;! stack_switching = true +;; Testing that the creation of the necessary payload buffers works as expect, +;; even when the same continuation reference is suspended multiple times + +(module + (type $unit_to_int (func (result i32))) + (type $int_to_unit (func (param i32))) + (type $int_to_int (func (param i32) (result i32))) + (type $2int_to_int (func (param i32 i32) (result i32))) + (type $3int_to_int (func (param i32 i32 i32) (result i32))) + + (type $ct0 (cont $int_to_int)) + (type $ct1 (cont $unit_to_int)) + (type $ct2 (cont $2int_to_int)) + + (global $checker (mut i32) (i32.const 0)) + + (func $check_stack (param $expected i32) (param $actual i32) (result i32) + (if (result i32) + (i32.xor (local.get $expected) (local.get $actual)) + (then (unreachable)) + (else (local.get $actual)))) + + (func $check_stack2 + (param $expected1 i32) + (param $expected2 i32) + (param $actual1 i32) + (param $actual2 i32) + (result i32 i32) + (if + (i32.xor (local.get $expected1) (local.get $actual1)) + (then (unreachable)) + (else)) + (if + (i32.xor (local.get $expected2) (local.get $actual2)) + (then (unreachable)) + (else)) + (local.get $actual1) + (local.get $actual2)) + + + (tag $e (param i32) (result i32)) + (tag $f (param i32 i32) (result i32 i32)) + + (func $g (param $x i32) (result i32) + (i32.add (local.get $x) (i32.const 1)) + (call $check_stack (i32.const 10)) + (suspend $e) + (call $check_stack (i32.const 15)) + (i32.add (i32.const 5)) + (call $check_stack (i32.const 20)) + (suspend $e) + (call $check_stack (i32.const 25)) + (i32.const 30) + (suspend $f) + (call $check_stack2 (i32.const 35) (i32.const 40)) + (i32.add)) + (elem declare func $g) + + (func $test (export "test") (result i32) + (local $k1 (ref $ct0)) + (local $k2 (ref $ct1)) + (local $k3 (ref $ct2)) + (local $i i32) + + (block $on_e1 (result i32 (ref $ct0)) + (i32.const 9) + (cont.new $ct0 (ref.func $g)) + (cont.bind $ct0 $ct1) ;; binding 9 here as value of parameter $x of $g + (resume $ct1 (on $e $on_e1)) + (unreachable)) + (local.set $k1) + (call $check_stack (i32.const 10)) + (i32.add (i32.const 5)) + (call $check_stack (i32.const 15)) + (cont.bind $ct0 $ct1 (local.get $k1)) ;; binding 15 + (local.set $k2) + + + (block $on_e2 (result i32 (ref $ct0)) + (resume $ct1 (on $e $on_e2) (local.get $k2)) + (unreachable)) + (local.set $k1) + (call $check_stack (i32.const 20)) + (i32.add (i32.const 5)) + (call $check_stack (i32.const 25)) + (cont.bind $ct0 $ct1 (local.get $k1)) ;; binding 25 + (local.set $k2) + (block $on_f (result i32 i32 (ref $ct2)) + (resume $ct1 (on $f $on_f) (local.get $k2)) + (unreachable)) + (local.set $k3) + (call $check_stack2 (i32.const 25) (i32.const 30)) + (i32.add (i32.const 10)) + (local.set $i) + (i32.add (i32.const 10)) + (local.get $i) + (call $check_stack2 (i32.const 35) (i32.const 40)) + (local.get $k3) + (cont.bind $ct2 $ct1) ;; binding 35, 40 + (resume $ct1)) +) + +(assert_return (invoke "test") (i32.const 75)) diff --git a/tests/misc_testsuite/stack-switching/cont_bind_null.wast b/tests/misc_testsuite/stack-switching/cont_bind_null.wast new file mode 100644 index 000000000000..91deab3bbd80 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_bind_null.wast @@ -0,0 +1,12 @@ +;;! stack_switching = true +(module + (type $ft (func)) + (type $ct (cont $ft)) + + (func $entry (export "entry") + (cont.bind $ct $ct (ref.null $ct)) + (drop) + ) +) + +(assert_trap (invoke "entry") "null reference") diff --git a/tests/misc_testsuite/stack-switching/cont_forwarding1.wast b/tests/misc_testsuite/stack-switching/cont_forwarding1.wast new file mode 100644 index 000000000000..1200ac2b8c5c --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_forwarding1.wast @@ -0,0 +1,50 @@ +;;! stack_switching = true +;; Simple forwarding, no payloads, no param or return values on +;; function, immediately resumed by handler + +(module + + (type $unit_to_unit (func)) + (type $ct (cont $unit_to_unit)) + + (type $g2_res_type (func (result (ref $ct)))) + (type $g2_res_type_ct (cont $g2_res_type)) + + (tag $e1) + (tag $e2) + + (global $marker (mut i32) (i32.const 0)) + + (func $update_marker (param $x i32) + (i32.add (global.get $marker) (i32.const 1)) + (i32.mul (local.get $x)) + (global.set $marker)) + + (func $g1 + (call $update_marker (i32.const 2)) + (suspend $e1) + (call $update_marker (i32.const 3))) + (elem declare func $g1) + + ;; Calls $g1 as continuation, but only handles e2 rather than e1 + (func $g2 + (block $on_e2 (result (ref $ct)) + (call $update_marker (i32.const 5)) + (resume $ct (on $e2 $on_e2) (cont.new $ct (ref.func $g1))) + (return)) + (unreachable)) + (elem declare func $g2) + + (func $g3 + (block $on_e1 (result (ref $ct)) + (call $update_marker (i32.const 7)) + (resume $ct (on $e1 $on_e1) (cont.new $ct (ref.func $g2))) + (unreachable)) + (call $update_marker (i32.const 11)) + (resume $ct)) + + (func $test (export "test") (result i32) + (call $g3) + (global.get $marker))) + +(assert_return (invoke "test") (i32.const 2742)) diff --git a/tests/misc_testsuite/stack-switching/cont_forwarding2.wast b/tests/misc_testsuite/stack-switching/cont_forwarding2.wast new file mode 100644 index 000000000000..91b4f21d04be --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_forwarding2.wast @@ -0,0 +1,40 @@ +;;! stack_switching = true +;; Like previous test, but with param and return values on functions (but no tag payloads) + +(module + + (type $int_to_int (func (param i32) (result i32))) + (type $unit_to_int (func (result i32))) + (type $ct0 (cont $int_to_int)) + (type $ct1 (cont $unit_to_int)) + + (tag $e1) + (tag $e2) + + (func $g1 (param $x i32) (result i32) + (suspend $e1) + (i32.add (local.get $x) (i32.const 1))) + (elem declare func $g1) + + ;; Calls $g1 as continuation, but only handles e2 rather than e1 + (func $g2 (param $x i32) (result i32) + (block $on_e2 (result (ref $ct1)) + (i32.add (local.get $x) (i32.const 1)) + (resume $ct0 (on $e2 $on_e2) (cont.new $ct0 (ref.func $g1))) + (i32.add (i32.const 1)) + (return)) + (unreachable)) + (elem declare func $g2) + + (func $g3 (param $x i32) (result i32) + (block $on_e1 (result (ref $ct1)) + (i32.add (local.get $x) (i32.const 1)) + (resume $ct0 (on $e1 $on_e1) (cont.new $ct0 (ref.func $g2))) + (unreachable)) + (resume $ct1) + (i32.add (i32.const 1))) + + (func $test (export "test") (result i32) + (call $g3 (i32.const 1)))) + +(assert_return (invoke "test") (i32.const 6)) diff --git a/tests/misc_testsuite/stack-switching/cont_forwarding3.wast b/tests/misc_testsuite/stack-switching/cont_forwarding3.wast new file mode 100644 index 000000000000..5ff2b41cf3de --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_forwarding3.wast @@ -0,0 +1,45 @@ +;;! stack_switching = true +;; Like previous test, but tag has payloads (param and return values) + +(module + + (type $int_to_int (func (param i32) (result i32))) + (type $unit_to_int (func (result i32))) + (type $ct0 (cont $int_to_int)) + (type $ct1 (cont $unit_to_int)) + + (tag $e1 (param i32) (result i32)) + (tag $e2) + + (func $g1 (param $x i32) (result i32) + (i32.add (local.get $x) (i32.const 1)) + (suspend $e1) + (i32.add (i32.const 1))) + (elem declare func $g1) + + ;; Calls $g1 as continuation, but only handles e2 rather than e1 + (func $g2 (param $x i32) (result i32) + (block $on_e2 (result (ref $ct1)) + (i32.add (local.get $x) (i32.const 1)) + (resume $ct0 (on $e2 $on_e2) (cont.new $ct0 (ref.func $g1))) + (i32.add (i32.const 1)) + (return)) + (unreachable)) + (elem declare func $g2) + + (func $g3 (param $x i32) (result i32) + (local $k (ref $ct0)) + (block $on_e1 (result i32 (ref $ct0)) + (i32.add (local.get $x) (i32.const 1)) + (resume $ct0 (on $e1 $on_e1) (cont.new $ct0 (ref.func $g2))) + (unreachable)) + (local.set $k) + (i32.add (i32.const 1)) + (local.get $k) + (resume $ct0) + (i32.add (i32.const 1))) + + (func $test (export "test") (result i32) + (call $g3 (i32.const 1)))) + +(assert_return (invoke "test") (i32.const 8)) diff --git a/tests/misc_testsuite/stack-switching/cont_forwarding4.wast b/tests/misc_testsuite/stack-switching/cont_forwarding4.wast new file mode 100644 index 000000000000..1f47b3198acc --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_forwarding4.wast @@ -0,0 +1,56 @@ +;;! stack_switching = true +;; Continuation is not immediately resumed, instead we run a different continuation in the meantime. + +(module + + (type $int_to_int (func (param i32) (result i32))) + (type $unit_to_int (func (result i32))) + (type $ct0 (cont $int_to_int)) + (type $ct1 (cont $unit_to_int)) + + (tag $e1 (param i32) (result i32)) + (tag $e2) + + (func $g1 (param $x i32) (result i32) + (i32.add (local.get $x) (i32.const 1)) + (suspend $e1) + (i32.add (i32.const 1))) + (elem declare func $g1) + + ;; Calls $g1 as continuation, but only handles e2 rather than e1 + (func $g2 (param $x i32) (result i32) + (block $on_e2 (result (ref $ct1)) + (i32.add (local.get $x) (i32.const 1)) + (resume $ct0 (on $e2 $on_e2) (cont.new $ct0 (ref.func $g1))) + (i32.add (i32.const 1)) + (return)) + (unreachable)) + (elem declare func $g2) + + (func $g3 (param $x i32) (result i32) + (local $k1 (ref $ct0)) + (local $k2 (ref $ct0)) + (block $on_e1 (result i32 (ref $ct0)) + (i32.add (local.get $x) (i32.const 1)) + (resume $ct0 (on $e1 $on_e1) (cont.new $ct0 (ref.func $g2))) + (unreachable)) + (local.set $k1) + (i32.add (i32.const 1)) + + ;; We run another continuation before resuming $k1 + (block $on_e1_2 (param i32) (result i32 (ref $ct0)) + (resume $ct0 (on $e1 $on_e1_2) (cont.new $ct0 (ref.func $g1))) + (unreachable)) + (local.set $k2) + (i32.add (i32.const 1)) + (resume $ct0 (local.get $k2)) + (i32.add (i32.const 1)) + + ;; Now finally resume $k1 + (resume $ct0 (local.get $k1)) + (i32.add (i32.const 1))) + + (func $test (export "test") (result i32) + (call $g3 (i32.const 1)))) + +(assert_return (invoke "test") (i32.const 12)) diff --git a/tests/misc_testsuite/stack-switching/cont_forwarding5.wast b/tests/misc_testsuite/stack-switching/cont_forwarding5.wast new file mode 100644 index 000000000000..f203197230ab --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_forwarding5.wast @@ -0,0 +1,58 @@ +;;! stack_switching = true +;; Continuation is not immediately resumed, we pass it to a different one as an +;; argument, increasing the length of the chain by adding yet another useless +;; handler. + +(module + + (type $int_to_int (func (param i32) (result i32))) + (type $unit_to_int (func (result i32))) + (type $ct0 (cont $int_to_int)) + (type $ct1 (cont $unit_to_int)) + + (type $ct1_to_int (func (param (ref $ct1)) (result i32))) + (type $ct2 (cont $ct1_to_int)) + + (tag $e1 (param i32) (result i32)) + (tag $e2) + + (func $g1 (param $x i32) (result i32) + (i32.add (local.get $x) (i32.const 1)) + (suspend $e1) + (i32.add (i32.const 1))) + (elem declare func $g1) + + ;; Calls $g1 as continuation, but only handles e2 rather than e1 + (func $g2 (param $x i32) (result i32) + (block $on_e2 (result (ref $ct1)) + (i32.add (local.get $x) (i32.const 1)) + (resume $ct0 (on $e2 $on_e2) (cont.new $ct0 (ref.func $g1))) + (i32.add (i32.const 1)) + (return)) + (unreachable)) + (elem declare func $g2) + + (func $g3 (param $k (ref $ct1)) (result i32) + (resume $ct1 (local.get $k))) + (elem declare func $g3) + + (func $g4 (param $x i32) (result i32) + (local $k1 (ref $ct0)) + (local $k2 (ref $ct1)) + (block $on_e1 (result i32 (ref $ct0)) + (i32.add (local.get $x) (i32.const 1)) + (resume $ct0 (on $e1 $on_e1) (cont.new $ct0 (ref.func $g2))) + (unreachable)) + (local.set $k1) + (i32.add (i32.const 1)) + (cont.bind $ct0 $ct1 (local.get $k1)) + (local.set $k2) + + ;; We resume $k1 by running it from within another continuation + (resume $ct2 (local.get $k2) (cont.new $ct2 (ref.func $g3))) + (i32.add (i32.const 1))) + + (func $test (export "test") (result i32) + (call $g4 (i32.const 1)))) + +(assert_return (invoke "test") (i32.const 8)) diff --git a/tests/misc_testsuite/stack-switching/cont_forwarding6.wast b/tests/misc_testsuite/stack-switching/cont_forwarding6.wast new file mode 100644 index 000000000000..df0bfdb24a6a --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_forwarding6.wast @@ -0,0 +1,56 @@ +;;! stack_switching = true +;; The resumed continuation suspends again (from the same inner function), using the same tag. +;; We install a new handler at the outermost level, which should be forwarded to correctly. + +(module + + (type $int_to_int (func (param i32) (result i32))) + (type $unit_to_int (func (result i32))) + (type $ct0 (cont $int_to_int)) + (type $ct1 (cont $unit_to_int)) + + (tag $e1 (param i32) (result i32)) + (tag $e2) + + (func $g1 (param $x i32) (result i32) + (i32.add (local.get $x) (i32.const 1)) + (suspend $e1) + (i32.add (i32.const 1)) + (suspend $e1) + (i32.add (i32.const 1))) + (elem declare func $g1) + + ;; Calls $g1 as continuation, but only handles e2 rather than e1 + (func $g2 (param $x i32) (result i32) + (block $on_e2 (result (ref $ct1)) + ;;(call $update_marker (i32.const 5)) + (i32.add (local.get $x) (i32.const 1)) + (resume $ct0 (on $e2 $on_e2) (cont.new $ct0 (ref.func $g1))) + (i32.add (i32.const 1)) + (return)) + (unreachable)) + (elem declare func $g2) + + (func $g3 (param $x i32) (result i32) + (local $k1 (ref $ct0)) + (block $on_e1 (result i32 (ref $ct0)) + (i32.add (local.get $x) (i32.const 1)) + (resume $ct0 (on $e1 $on_e1) (cont.new $ct0 (ref.func $g2))) + (unreachable)) + (local.set $k1) + (i32.add (i32.const 1)) + + ;; g1 will suspend again. We install a new handler for $e1 + (block $on_e1_2 (param i32) (result i32 (ref $ct0)) + (resume $ct0 (on $e1 $on_e1_2) (local.get $k1)) + (unreachable)) + (local.set $k1) + (i32.add (i32.const 1)) + + (resume $ct0 (local.get $k1)) + (i32.add (i32.const 1))) + + (func $test (export "test") (result i32) + (call $g3 (i32.const 1)))) + +(assert_return (invoke "test") (i32.const 10)) diff --git a/tests/misc_testsuite/stack-switching/cont_forwarding7.wast b/tests/misc_testsuite/stack-switching/cont_forwarding7.wast new file mode 100644 index 000000000000..167b941f0d20 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_forwarding7.wast @@ -0,0 +1,54 @@ +;;! stack_switching = true +;; The resumed continuation suspends again (but not from the same function), using the same tag. +;; We use the same outer handler in a loop + +(module + + (type $int_to_int (func (param i32) (result i32))) + (type $unit_to_int (func (result i32))) + (type $ct0 (cont $int_to_int)) + (type $ct1 (cont $unit_to_int)) + + (tag $e1 (param i32) (result i32)) + (tag $e2) + + (func $g1 (param $x i32) (result i32) + (i32.add (local.get $x) (i32.const 1)) + (suspend $e1) + (i32.add (i32.const 1)) + (suspend $e1) + (i32.add (i32.const 1))) + (elem declare func $g1) + + ;; Calls $g1 as continuation, but only handles e2 rather than e1 + (func $g2 (param $x i32) (result i32) + (block $on_e2 (result (ref $ct1)) + (i32.add (local.get $x) (i32.const 1)) + (resume $ct0 (on $e2 $on_e2) (cont.new $ct0 (ref.func $g1))) + (i32.add (i32.const 1)) + ;; suspend to $e1 again, this time from the direct child of the handler of $e1 + (suspend $e1) + (i32.add (i32.const 1)) + (return)) + (unreachable)) + (elem declare func $g2) + + (func $g3 (param $x i32) (result i32) + (local $k1 (ref $ct0)) + (local.get $x) + (cont.new $ct0 (ref.func $g2)) + + (loop $loop (param i32 (ref $ct0)) + (block $on_e1 (param i32 (ref $ct0)) (result i32 (ref $ct0)) + (resume $ct0 (on $e1 $on_e1)) + (return)) + (local.set $k1) + (i32.add (i32.const 1)) + (local.get $k1) + (br $loop)) + (unreachable)) + + (func $test (export "test") (result i32) + (call $g3 (i32.const 1)))) + +(assert_return (invoke "test") (i32.const 10)) diff --git a/tests/misc_testsuite/stack-switching/cont_forwarding8.wast b/tests/misc_testsuite/stack-switching/cont_forwarding8.wast new file mode 100644 index 000000000000..604cdd2b5d88 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_forwarding8.wast @@ -0,0 +1,63 @@ +;;! stack_switching = true +;; The resumed continuation suspends again, to the same and different tags. +;; We use the same outer handler in a loop + +(module + + (type $int_to_int (func (param i32) (result i32))) + (type $unit_to_int (func (result i32))) + (type $ct0 (cont $int_to_int)) + + (tag $e1 (param i32) (result i32)) + (tag $e2 (param i32) (result i32)) + + (func $g1 (param $x i32) (result i32) + (i32.add (local.get $x) (i32.const 1)) + (suspend $e1) + (i32.add (i32.const 1)) + (suspend $e2) + (i32.add (i32.const 1)) + (suspend $e1) + (i32.add (i32.const 1))) + (elem declare func $g1) + + ;; Calls $g1 as continuation, but only handles e2 rather than e1 + (func $g2 (param $x i32) (result i32) + (local $k1 (ref $ct0)) + (block $on_e2 (result i32 (ref $ct0)) + (i32.add (local.get $x) (i32.const 1)) + (resume $ct0 (on $e2 $on_e2) (cont.new $ct0 (ref.func $g1))) + (unreachable)) + (local.set $k1) + + ;; we expect to suspend to $e2 eventually, at which point we install a new handler, while keeping the + ;; "outer" one for $e1 intact. + (i32.add (i32.const 1)) + (block $on_e2_2 (param i32) (result i32 (ref $ct0)) + (resume $ct0 (on $e2 $on_e2_2) (local.get $k1)) + (i32.add (i32.const 1)) + (suspend $e1) + (i32.add (i32.const 1)) + (return)) + (unreachable)) + (elem declare func $g2) + + (func $g3 (param $x i32) (result i32) + (local $k1 (ref $ct0)) + (local.get $x) + (cont.new $ct0 (ref.func $g2)) + + (loop $loop (param i32 (ref $ct0)) + (block $on_e1 (param i32 (ref $ct0)) (result i32 (ref $ct0)) + (resume $ct0 (on $e1 $on_e1)) + (return)) + (local.set $k1) + (i32.add (i32.const 1)) + (local.get $k1) + (br $loop)) + (unreachable)) + + (func $test (export "test") (result i32) + (call $g3 (i32.const 1)))) + +(assert_return (invoke "test") (i32.const 12)) diff --git a/tests/misc_testsuite/stack-switching/cont_nary.wast b/tests/misc_testsuite/stack-switching/cont_nary.wast new file mode 100644 index 000000000000..cb91f999fb61 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_nary.wast @@ -0,0 +1,90 @@ +;;! stack_switching = true +;; Tests using support for n-ary continuations +;; Uses a function as continuation that has 3 param and 5 return values +;; Uses tag that has 4 elements of payloads +;; All of these mix i32 and i64 values + + +(module + (type $unit_to_unit (func)) + (type $unit_to_int (func (result i32))) + (type $int_to_unit (func (param i32))) + (type $int_to_int (func (param i32) (result i32))) + + + ;; type of function f + (type $f_t + (func + (param i64 i32 i64) + (result i32 i64 i32 i64 i32))) + (type $f_ct (cont $f_t)) + + + ;; type of the resumption we expect to see in our handler for $e + (type $res_ft + (func + (param i64 i32 i64 i32) + (result i32 i64 i32 i64 i32))) + (type $res (cont $res_ft)) + + ;; This is 10^10, which exceeds what can be stored in any 32 bit type + (global $big i64 (i64.const 10_000_000_000)) + + (tag $e + (param i32 i64 i32 i64) + (result i64 i32 i64 i32)) + + + + (func $f (export "f") + (param $x i64) (param $y i32) (param $z i64) + (result i32 i64 i32 i64 i32) + + ;; value to stay on the stack as part of return values + (i32.const 1) + + ;; values to be passed to $e + (i32.const 10) + (local.get $z) + (local.get $y) + (local.get $x) + (suspend $e) + ) + + (func $test (export "test") (result i32 i64) + (local $i64_acc i64) + (local $i32_acc i32) + (local $k (ref $res)) + (local.set $i64_acc (i64.const 0)) + (local.set $i32_acc (i32.const 0)) + + + (block $on_e (result i32 i64 i32 i64 (ref $res)) ;; lets call these values v1 v2 v3 v4 k + (global.get $big) + (i32.const 100) + (i64.mul (global.get $big) (i64.const 10)) + (resume $f_ct (on $e $on_e) (cont.new $f_ct (ref.func $f))) + (unreachable)) + ;; after on_e + (local.set $k) + (i32.const 1000) + ;; We pass v2 v3 v4 123 as arguments to the continuation, leaving v1 on the stack + (resume $res (local.get $k)) + ;; We now have v1 and the five return values of $f on the stack, i.e. [i32 i32 i64 i32 i64 i32] + ;; Lets accumulate them + ;; + + (local.set $i32_acc (i32.add (local.get $i32_acc))) + (local.set $i64_acc (i64.add (local.get $i64_acc))) + (local.set $i32_acc (i32.add (local.get $i32_acc))) + (local.set $i64_acc (i64.add (local.get $i64_acc))) + (local.set $i32_acc (i32.add (local.get $i32_acc))) + (local.set $i32_acc (i32.add (local.get $i32_acc))) + + ;; ;; Set up return values + (local.get $i32_acc) + (local.get $i64_acc)) + +) + +(assert_return (invoke "test") (i32.const 1111) (i64.const 110_000_000_000)) diff --git a/tests/misc_testsuite/stack-switching/cont_nested1.wast b/tests/misc_testsuite/stack-switching/cont_nested1.wast new file mode 100644 index 000000000000..dbac978641c9 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_nested1.wast @@ -0,0 +1,53 @@ +;;! stack_switching = true +;; test using continuations from within a continuation + +(module + + (type $unit_to_unit (func)) + (type $ct (cont $unit_to_unit)) + + (type $g2_res_type (func (result (ref $ct)))) + (type $g2_res_type_ct (cont $g2_res_type)) + + (tag $e1) + (tag $e2 (param (ref $ct))) + + (global $marker (mut i32) (i32.const 0)) + + (func $update_marker (param $x i32) + (i32.add (global.get $marker) (i32.const 1)) + (i32.mul (local.get $x)) + (global.set $marker)) + + (func $g1 + (call $update_marker (i32.const 2)) + (suspend $e1) + (call $update_marker (i32.const 3)) + ) + (elem declare func $g1) + + (func $g2 + (local $k1 (ref $ct)) + (local $k2 (ref $ct)) + (call $update_marker (i32.const 5)) + + (block $on_e1 (result (ref $ct)) + (resume $ct (on $e1 $on_e1) (cont.new $ct (ref.func $g1))) + (unreachable)) + (local.set $k1) + (call $update_marker (i32.const 7)) + (block $on_e1_2 (result (ref $ct)) + (resume $ct (on $e1 $on_e1_2) (cont.new $ct (ref.func $g1))) + (unreachable)) + (local.set $k2) + (call $update_marker (i32.const 11)) + (resume $ct (local.get $k1))) + (elem declare func $g2) + + + + (func $test (export "test") (result i32) + (resume $ct (cont.new $ct (ref.func $g2))) + (global.get $marker))) + +(assert_return (invoke "test") (i32.const 6_108)) diff --git a/tests/misc_testsuite/stack-switching/cont_nested2.wast b/tests/misc_testsuite/stack-switching/cont_nested2.wast new file mode 100644 index 000000000000..5d4a6c6c6cdb --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_nested2.wast @@ -0,0 +1,65 @@ +;;! stack_switching = true +;; Similar to cont_nested1, but with payloads + +(module + + (type $int_to_int (func (param i32) (result i32))) + (type $ct (cont $int_to_int)) + + (type $unit_to_int (func (result i32))) + (type $ct_unit_to_int (cont $unit_to_int)) + + (tag $e1 (param i32) (result i32)) + + (global $marker (mut i32) (i32.const 0)) + + ;; (func $update_marker (param $x i32) (result i32) + ;; (i32.add (global.get $marker) (i32.const 1)) + ;; (i32.mul (local.get $x)) + ;; (global.set $marker) + ;; (global.get $marker)) + + (func $scramble (param $x i32) (param $y i32) (result i32) + (i32.add (local.get $y) (i32.const 1)) + (i32.mul (local.get $x)) + ) + + (func $g1 (param $x i32) (result i32) + (call $scramble (i32.const 3) (local.get $x)) + (suspend $e1) + (call $scramble (i32.const 5)) + (i32.add (local.get $x)) + (global.set $marker) + (global.get $marker)) + (elem declare func $g1) + + (func $g2 (result i32) + (local $k1 (ref $ct)) + (local $v i32) + + (block $on_e1 (result i32 (ref $ct)) + (resume $ct (on $e1 $on_e1) (i32.const 7) (cont.new $ct (ref.func $g1))) + (unreachable)) + (local.set $k1) + (call $scramble (i32.const 11)) ;; scramble the value received via $e1 from $g1 + (local.set $v) + + (block $on_e1_2 (result i32 (ref $ct)) + (resume $ct (on $e1 $on_e1_2) (local.get $v) (cont.new $ct (ref.func $g1))) + (unreachable)) + (drop) ;; drop continuation, we don't intend to resume the second invocation of g1 + (call $scramble (i32.const 13)) + + (resume $ct (local.get $k1)) + (i32.add (global.get $marker))) + (elem declare func $g2) + + + + (func $test (export "test") (result i32) + (resume $ct_unit_to_int (cont.new $ct_unit_to_int (ref.func $g2))) + + ) + ) + +(assert_return (invoke "test") (i32.const 145_670)) diff --git a/tests/misc_testsuite/stack-switching/cont_nested3.wast b/tests/misc_testsuite/stack-switching/cont_nested3.wast new file mode 100644 index 000000000000..5e2c370db4d1 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_nested3.wast @@ -0,0 +1,41 @@ +;;! stack_switching = true +;; Minimal test for resuming continuation after its original parent is gone + +(module + + (type $unit_to_unit (func)) + (type $ct (cont $unit_to_unit)) + + (type $g2 (func (result (ref $ct)))) + (type $g2_ct (cont $g2)) + + (tag $e1) + ;;(tag $e2 (param (ref $ct))) + + (global $marker (mut i32) (i32.const 0)) + + ;;(global $orphan (mut (ref $ct))) + + (func $g1 + (suspend $e1) + (global.set $marker (i32.const 100)) + ) + (elem declare func $g1) + + (func $g2 (result (ref $ct)) + (block $on_e1 (result (ref $ct)) + (resume $ct (on $e1 $on_e1) (cont.new $ct (ref.func $g1))) + (unreachable)) + ;; continuation becomes return value + ) + + (elem declare func $g2) + + + (func $test (export "test") (result i32) + (resume $g2_ct (cont.new $g2_ct (ref.func $g2))) + (resume $ct) ;; resume return value of $g2 + (global.get $marker)) + ) + +(assert_return (invoke "test") (i32.const 100)) diff --git a/tests/misc_testsuite/stack-switching/cont_nested4.wast b/tests/misc_testsuite/stack-switching/cont_nested4.wast new file mode 100644 index 000000000000..281be654e5f4 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_nested4.wast @@ -0,0 +1,45 @@ +;;! stack_switching = true +;; Minimal test for resuming continuation after its original parent is suspended + +(module + + (type $unit_to_unit (func)) + (type $ct (cont $unit_to_unit)) + + ;;(type $g2 (func (result (ref $ct)))) + ;;(type $g2_ct (cont $g2)) + + (tag $e1) + (tag $e2 (param (ref $ct))) + + (global $marker (mut i32) (i32.const 0)) + + ;;(global $orphan (mut (ref $ct))) + + (func $g1 + (suspend $e1) + (global.set $marker (i32.const 100)) + ) + (elem declare func $g1) + + (func $g2 + (block $on_e1 (result (ref $ct)) + (resume $ct (on $e1 $on_e1) (cont.new $ct (ref.func $g1))) + (unreachable)) + (suspend $e2) + ;; continuation becomes return value + (unreachable)) + + (elem declare func $g2) + + + (func $test (export "test") (result i32) + (block $on_e2 (result (ref $ct) (ref $ct)) + (resume $ct (on $e2 $on_e2) (cont.new $ct (ref.func $g2))) + (unreachable)) + (drop) ;; drop the continuation (i.e., for resuming g2) + (resume $ct) ;; resume continuation received as payload of $e2 (i.e., continuing execution of $g1) + (global.get $marker)) +) + +(assert_return (invoke "test") (i32.const 100)) diff --git a/tests/misc_testsuite/stack-switching/cont_nested5.wast b/tests/misc_testsuite/stack-switching/cont_nested5.wast new file mode 100644 index 000000000000..0d8033edf9f5 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_nested5.wast @@ -0,0 +1,69 @@ +;;! stack_switching = true +;; test using continuations from within a continuation + + +(module + + (type $unit_to_unit (func)) + (type $ct (cont $unit_to_unit)) + + (type $g2_res_type (func (result (ref $ct)))) + (type $g2_res_type_ct (cont $g2_res_type)) + + (tag $e1) + (tag $e2 (param (ref $ct))) + + (global $marker (mut i32) (i32.const 0)) + + (func $update_marker (param $x i32) + (i32.add (global.get $marker) (i32.const 1)) + (i32.mul (local.get $x)) + (global.set $marker)) + + (func $g1 + (call $update_marker (i32.const 2)) + (suspend $e1) + (call $update_marker (i32.const 3)) + ) + (elem declare func $g1) + + (func $g2 (result (ref $ct)) + (local $k1 (ref $ct)) + (local $k2 (ref $ct)) + (call $update_marker (i32.const 5)) + + (block $on_e1 (result (ref $ct)) + (resume $ct (on $e1 $on_e1) (cont.new $ct (ref.func $g1))) + (unreachable)) + (local.set $k1) + (call $update_marker (i32.const 7)) + (block $on_e1_2 (result (ref $ct)) + (resume $ct (on $e1 $on_e1_2) (cont.new $ct (ref.func $g1))) + (unreachable)) + (local.set $k2) + (call $update_marker (i32.const 11)) + (resume $ct (local.get $k1)) + (call $update_marker (i32.const 13)) + (local.get $k2) + ) + (elem declare func $g2) + + (func $g3 + (call $update_marker (i32.const 17)) + (resume $g2_res_type_ct (cont.new $g2_res_type_ct (ref.func $g2))) + (call $update_marker (i32.const 19)) + (suspend $e2)) + (elem declare func $g3) + + + (func $test (export "test") (result i32) + (call $update_marker (i32.const 23)) + (block $on_e2 (result (ref $ct) (ref $ct)) + (resume $ct (on $e2 $on_e2) (cont.new $ct (ref.func $g3))) + (unreachable)) + (drop) ;; we won't resume g3, but want the payload + (call $update_marker (i32.const 31)) + (resume $ct) + (global.get $marker))) + +(assert_return (invoke "test") (i32.const 490_074_902)) diff --git a/tests/misc_testsuite/stack-switching/cont_nested6.wast b/tests/misc_testsuite/stack-switching/cont_nested6.wast new file mode 100644 index 000000000000..c1c559f26dac --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_nested6.wast @@ -0,0 +1,45 @@ +;;! stack_switching = true +;; test proper handling of TSP pointer after a continuation returns normally + +(module + + (type $unit_to_unit (func)) + (type $ct (cont $unit_to_unit)) + + (type $g2_res_type (func (result (ref $ct)))) + (type $g2_res_type_ct (cont $g2_res_type)) + + (tag $e1) + + (global $marker (mut i32) (i32.const 0)) + + (func $update_marker (param $x i32) + (i32.add (global.get $marker) (i32.const 1)) + (i32.mul (local.get $x)) + (global.set $marker)) + + (func $g1 + (call $update_marker (i32.const 2))) + (elem declare func $g1) + + (func $g2 + (call $update_marker (i32.const 3)) + + (resume $ct (cont.new $ct (ref.func $g1))) + (call $update_marker (i32.const 5)) + + ;; This suspend only works correctly if we reset the TSP + ;; pointer after the g1 continuation returned. + (suspend $e1)) + + (elem declare func $g2) + + + (func $test (export "test") (result i32) + (block $on_e1 (result (ref $ct)) + (resume $ct (on $e1 $on_e1) (cont.new $ct (ref.func $g2))) + (unreachable)) + (drop) + (global.get $marker))) + +(assert_return (invoke "test") (i32.const 45)) diff --git a/tests/misc_testsuite/stack-switching/cont_new.wast b/tests/misc_testsuite/stack-switching/cont_new.wast new file mode 100644 index 000000000000..026df72a74f3 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_new.wast @@ -0,0 +1,17 @@ +;;! stack_switching = true +(module + (type $ft (func)) + (type $ct (cont $ft)) + + (func $noop) + (elem declare func $noop) + + (func $make-cont (result (ref $ct)) + (cont.new $ct (ref.func $noop))) + + (func $f (export "f") (result i32) + (call $make-cont) + (ref.is_null)) +) + +(assert_return (invoke "f") (i32.const 0)) \ No newline at end of file diff --git a/tests/misc_testsuite/stack-switching/cont_new_null.wast b/tests/misc_testsuite/stack-switching/cont_new_null.wast new file mode 100644 index 000000000000..75e7429997cf --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_new_null.wast @@ -0,0 +1,12 @@ +;;! stack_switching = true +(module + (type $ft (func)) + (type $ct (cont $ft)) + + (func $entry (export "entry") + (cont.new $ct (ref.null $ft)) + (drop) + ) +) + +(assert_trap (invoke "entry") "null reference") diff --git a/tests/misc_testsuite/stack-switching/cont_resume.wast b/tests/misc_testsuite/stack-switching/cont_resume.wast new file mode 100644 index 000000000000..af036a46274b --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_resume.wast @@ -0,0 +1,18 @@ +;;! stack_switching = true +(module + (type $ft (func)) + (type $ct (cont $ft)) + + (global $i (mut i32) (i32.const 0)) + + (func $g + (global.set $i (i32.const 42))) + (elem declare func $g) + + (func $f (export "f") (result i32) + (global.set $i (i32.const 99)) + (resume $ct (cont.new $ct (ref.func $g))) + (global.get $i)) +) + +(assert_return (invoke "f") (i32.const 42)) \ No newline at end of file diff --git a/tests/misc_testsuite/stack-switching/cont_resume1.wast b/tests/misc_testsuite/stack-switching/cont_resume1.wast new file mode 100644 index 000000000000..a81588d5c84f --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_resume1.wast @@ -0,0 +1,28 @@ +;;! stack_switching = true +(module + (type $ft_init (func)) + (type $ct_init (cont $ft_init)) + (type $ft (func (param i32))) + (type $ct (cont $ft)) + (tag $yield (result i32)) + + (global $i (mut i32) (i32.const 0)) + + (func $g + (suspend $yield) + (global.set $i)) + (elem declare func $g) + + (func $f (export "f") (result i32) + (local $k (ref null $ct)) + (global.set $i (i32.const 99)) + (block $on_yield (result (ref $ct)) + (resume $ct_init (on $yield $on_yield) (cont.new $ct_init (ref.func $g))) + (unreachable)) + ;; on_yield + (local.set $k) + (resume $ct (i32.const 42) (local.get $k)) + (global.get $i)) +) + +(assert_return (invoke "f") (i32.const 42)) \ No newline at end of file diff --git a/tests/misc_testsuite/stack-switching/cont_resume2.wast b/tests/misc_testsuite/stack-switching/cont_resume2.wast new file mode 100644 index 000000000000..b3f21da94138 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_resume2.wast @@ -0,0 +1,65 @@ +;;! stack_switching = true +;; This test requires the following to work: +;; 1. Passing arguments to tags and receiving values back at suspend sites +;; 2. Passing values to a continuation obtained from a handler +;; 3. Receving values once a resumed continuation returns oridinarly + +;; Not tested: Passing values when resuming a continuation obtained from +;; cont.new rather than a handler + +;; TODO(frank-emrich) Replace this with more fine-grained tests in the future + +(module + + (type $int_to_int (func (param i32) (result i32))) + + (type $cont_int_to_int (cont $int_to_int)) + + + (type $g_type (func (result i32))) + (type $g_ct (cont $g_type)) + + + (tag $e0 (param i32) (result i32)) ;; never actually invoked + (tag $e1 (param i32) (result i32)) + (tag $e2 (param i32) (result i32)) + (tag $e3 (param i32) (result i32)) ;; never actually invoked + + + (func $g (result i32) + (suspend $e1 (i32.const 42)) + (suspend $e2) ;; passes value obtained from doing $e on to $f + (i32.add (i32.const 21))) + (elem declare func $g) + + (func $f (export "f") (result i32) + (local $c (ref $cont_int_to_int)) + (block $on_e0_e2_e3 (result i32 (ref $cont_int_to_int)) + (block $on_e1 (result i32 (ref $cont_int_to_int)) + ;; We know that $e0, e2, e3 won't actually be performed here, but add a handler + ;; to test the switching logic + (resume $g_ct + (on $e0 $on_e0_e2_e3) + (on $e1 $on_e1) + (on $e2 $on_e0_e2_e3) + (on $e3 $on_e0_e2_e3) + (cont.new $g_ct (ref.func $g))) + (unreachable)) + ;; after $on_e1 + (local.set $c) + ;; stack now contains the value that $g passed to $e1, we manipulate it + (i32.add (i32.const 13)) + (local.get $c) + (resume $cont_int_to_int (on $e2 $on_e0_e2_e3)) + (unreachable)) + ;; after $on_e0_e2_e3 + ;; stack contains value that $g passed to $e2 and continuation + ;; We manipulate the value again before resuming the continuation + (local.set $c) + (i32.add (i32.const 24)) + (local.get $c) + (resume $cont_int_to_int) + )) + + +(assert_return (invoke "f") (i32.const 100)) diff --git a/tests/misc_testsuite/stack-switching/cont_resume_null.wast b/tests/misc_testsuite/stack-switching/cont_resume_null.wast new file mode 100644 index 000000000000..2df6cd627f77 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_resume_null.wast @@ -0,0 +1,11 @@ +;;! stack_switching = true +(module + (type $ft (func)) + (type $ct (cont $ft)) + + (func $entry (export "entry") + (resume $ct (ref.null $ct)) + ) +) + +(assert_trap (invoke "entry") "null reference") diff --git a/tests/misc_testsuite/stack-switching/cont_resume_return.wast b/tests/misc_testsuite/stack-switching/cont_resume_return.wast new file mode 100644 index 000000000000..d7674e8b4fa6 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_resume_return.wast @@ -0,0 +1,14 @@ +;;! stack_switching = true +(module + (type $ft (func (result i32))) + (type $ct (cont $ft)) + + (func $g (result i32) + (i32.const 42)) + (elem declare func $g) + + (func $f (export "f") (result i32) + (resume $ct (cont.new $ct (ref.func $g)))) +) + +(assert_return (invoke "f") (i32.const 42)) \ No newline at end of file diff --git a/tests/misc_testsuite/stack-switching/cont_return.wast b/tests/misc_testsuite/stack-switching/cont_return.wast new file mode 100644 index 000000000000..dde5ff643c4d --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_return.wast @@ -0,0 +1,17 @@ +;;! stack_switching = true +;; Test returning value from continuation function without any suspending + +(module + + (type $g_type (func (result i32))) + (type $g_ct (cont $g_type)) + + (func $g (result i32) + (i32.const 100)) + (elem declare func $g) + + (func $f (export "f") (result i32) + (resume $g_ct + (cont.new $g_ct (ref.func $g))))) + +(assert_return (invoke "f") (i32.const 100)) diff --git a/tests/misc_testsuite/stack-switching/cont_suspend.wast b/tests/misc_testsuite/stack-switching/cont_suspend.wast new file mode 100644 index 000000000000..ed3229c77bfc --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_suspend.wast @@ -0,0 +1,26 @@ +;;! stack_switching = true +;; Small continuation resume test +;; expected output: +;; 1 : i32 +;; 2 : i32 +;; 3 : i32 +(module + (func $print (import "spectest" "print_i32") (param i32) (result)) + (type $ft (func)) + (type $ct (cont $ft)) + (tag $h) + (func $f (export "f") + (suspend $h) + (call $print (i32.const 2))) + (func (export "run") (result i32) + (call $print (i32.const 1)) + (block $on_h (result (ref $ct)) + (resume $ct (on $h $on_h) + (cont.new $ct (ref.func $f))) + (unreachable)) + (drop) + (call $print (i32.const 3)) + (return (i32.const 42))) +) + +(assert_return (invoke "run") (i32.const 42)) diff --git a/tests/misc_testsuite/stack-switching/cont_table.wast b/tests/misc_testsuite/stack-switching/cont_table.wast new file mode 100644 index 000000000000..9f64edeb1cd8 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_table.wast @@ -0,0 +1,87 @@ +;;! stack_switching = true +(module + (type $ft (func (param i32) (result i32))) + (type $ct (cont $ft)) + + (table $conts0 0 (ref null $ct)) + (table $conts1 0 (ref null $ct)) + + (func $f (param $x i32) (result i32) + (i32.add (local.get $x) (i32.const 1)) + ) + (elem declare func $f) + + (func (export "table0_size") (result i32) + (table.size $conts0) + ) + + (func (export "table0_grow_f") (result i32) + (table.grow $conts0 (cont.new $ct (ref.func $f)) (i32.const 10)) + ) + (func (export "table1_grow_f") (result i32) + (table.grow $conts1 (cont.new $ct (ref.func $f)) (i32.const 10)) + ) + + (func (export "table_copy_0_to_1") + (table.copy $conts1 $conts0 (i32.const 0) (i32.const 0) (i32.const 10)) + ) + + (func (export "table0_fill_f") + (table.fill $conts0 (i32.const 0) (cont.new $ct (ref.func $f)) (i32.const 10)) + ) + + (func (export "table0_null_at") (param $i i32) (result i32) + (ref.is_null (table.get $conts0 (local.get $i))) + ) + (func (export "table1_null_at") (param $i i32) (result i32) + (ref.is_null (table.get $conts1 (local.get $i))) + ) + + (func (export "table0_set_f") (param $i i32) + (table.set $conts0 (local.get $i) (cont.new $ct (ref.func $f))) + ) + + (func (export "table0_set_null") (param $i i32) + (table.set $conts0 (local.get $i) (ref.null $ct)) + ) + + (func (export "table0_run") (param $i i32) (result i32) + (resume $ct (i32.const 99) (table.get $conts0 (local.get $i))) + ) + (func (export "table1_run") (param $i i32) (result i32) + (resume $ct (i32.const 99) (table.get $conts1 (local.get $i))) + ) +) + +(assert_return (invoke "table0_size") (i32.const 0)) +(assert_return (invoke "table0_grow_f") (i32.const 0)) +(assert_return (invoke "table0_size") (i32.const 10)) +;; At this point table 0 contains reference to the same, resumeable continuation +;; at all indices + + +;; We now consume the continuation, do table0[0] := null and write a fresh +;; continuation to table0[9], which we then consume +(assert_return (invoke "table0_run" (i32.const 0)) (i32.const 100)) +(assert_return (invoke "table0_null_at" (i32.const 9)) (i32.const 0)) +(assert_return (invoke "table0_set_f" (i32.const 9))) +(assert_return (invoke "table0_run" (i32.const 9)) (i32.const 100)) + + +;; Refill table with references to the same continuation, consume it, and do table0[9] := null +(invoke "table0_fill_f") +(assert_return (invoke "table0_run" (i32.const 0)) (i32.const 100)) +(assert_return (invoke "table0_set_f" (i32.const 0))) +(assert_return (invoke "table0_set_null" (i32.const 9))) + + +;; fill table1 with references to the same continuation running f and consume it +(invoke "table1_grow_f") +(assert_return (invoke "table1_run" (i32.const 0)) (i32.const 100)) + +;; We copy table0 to table1, meaning that table1[0] should contain an available +;; continuation, indices 1 to 8 contain the same consumed one, and table1[9] +;; contains null. +(invoke "table_copy_0_to_1") +(assert_return (invoke "table1_run" (i32.const 0)) (i32.const 100)) +(assert_return (invoke "table1_null_at" (i32.const 9)) (i32.const 1)) diff --git a/tests/misc_testsuite/stack-switching/cont_twice.wast b/tests/misc_testsuite/stack-switching/cont_twice.wast new file mode 100644 index 000000000000..60aad303667b --- /dev/null +++ b/tests/misc_testsuite/stack-switching/cont_twice.wast @@ -0,0 +1,16 @@ +;;! stack_switching = true +(module + (type $ft (func)) + (type $ct (cont $ft)) + + (func $f) + (func (export "resume_twice") + (local $k (ref $ct)) + (local.set $k (cont.new $ct (ref.func $f))) + (resume $ct (local.get $k)) + (resume $ct (local.get $k)) + ) + (elem declare func $f) +) + +(assert_trap (invoke "resume_twice") "continuation already consumed") \ No newline at end of file diff --git a/tests/misc_testsuite/stack-switching/dup.wast b/tests/misc_testsuite/stack-switching/dup.wast new file mode 100644 index 000000000000..937ab5910070 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/dup.wast @@ -0,0 +1,27 @@ +;;! stack_switching = true +(module + (type $ft (func)) + (type $ct (cont $ft)) + + (tag $t) + + (func $f + (suspend $t)) + (elem declare func $f) + + (func $dup (export "dup") (result i32) + (block $on_t-1 (result (ref $ct)) + (block $on_t-2 (result (ref $ct)) + (resume $ct (on $t $on_t-1) + ;;(on $t $on_t-2) + (cont.new $ct (ref.func $f))) + (return (i32.const 128)) + ) ;; on_t-2 [ (ref $ct) ] + (drop) + (return (i32.const 256)) + ) ;; on_t-1 [ (ref $ct) ] + (drop) + (return (i32.const 512))) +) + +(assert_return (invoke "dup") (i32.const 512)) \ No newline at end of file diff --git a/tests/misc_testsuite/stack-switching/linking_tags1.wast b/tests/misc_testsuite/stack-switching/linking_tags1.wast new file mode 100644 index 000000000000..9f9ed0def096 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/linking_tags1.wast @@ -0,0 +1,37 @@ +;;! stack_switching = true +(module $alien + (tag $alien_tag (export "alien_tag")) +) +(register "alien") + +(module $mine + (type $ft (func)) + (type $ct (cont $ft)) + (tag $alien_tag (import "alien" "alien_tag")) + (tag $my_tag) + (func $do_alien_tag + (suspend $alien_tag)) + + ;; Don't handle the imported alien. + (func (export "main-1") + (block $on_my_tag (result (ref $ct)) + (resume $ct (on $my_tag $on_my_tag) (cont.new $ct (ref.func $do_alien_tag))) + (unreachable) + ) + (unreachable)) + + ;; Handle the imported alien. + (func (export "main-2") + (block $on_alien_tag (result (ref $ct)) + (resume $ct (on $alien_tag $on_alien_tag) (cont.new $ct (ref.func $do_alien_tag))) + (unreachable) + ) + (drop)) + + (elem declare func $do_alien_tag) +) +(register "mine") +(assert_return (invoke "main-2")) +;; Due to issue #253, we need to make sure that nothing happens afterwards in +;; the test: +(assert_suspension (invoke "main-1") "unhandled") diff --git a/tests/misc_testsuite/stack-switching/linking_tags2.wast b/tests/misc_testsuite/stack-switching/linking_tags2.wast new file mode 100644 index 000000000000..dcba7349a621 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/linking_tags2.wast @@ -0,0 +1,136 @@ +;;! stack_switching = true +(module $foo + (type $ft (func (result i32))) + (type $ct (cont $ft)) + (type $ft-2 (func (param i32) (result i32))) + (type $ct-2 (cont $ft-2)) + + (tag $foo (export "foo") (result i32)) ;; occupies defined tag entry 0 + + (func $do_foo (export "do_foo") (result i32) + (suspend $foo)) + (func $handle_foo (export "handle_foo") (param $f (ref $ft)) (result i32) + (block $on_foo (result (ref $ct-2)) + (resume $ct (on $foo $on_foo) (cont.new $ct (local.get $f))) + (return) + ) ;; on_foo + (drop) + (return (i32.const 1)) + ) + (func (export "test_foo") (result i32) + (call $handle_foo (ref.func $do_foo))) + (elem declare func $do_foo) +) +(register "foo") +(assert_return (invoke "test_foo") (i32.const 1)) + +(module $baz + (type $ft (func (result i32))) + (type $ct (cont $ft)) + + (type $ft-2 (func (param i32) (result i32))) + (type $ct-2 (cont $ft-2)) + + (func $handle_foo (import "foo" "handle_foo") (param (ref $ft)) (result i32)) + (func $do_foo (import "foo" "do_foo") (result i32)) + + (tag $baz (result i32)) ;; unused, but occupies defined tag entry 0 + + (func $handle_baz (param $f (ref $ft)) (result i32) + (block $on_baz (result (ref $ct-2)) + (resume $ct (on $baz $on_baz) (cont.new $ct (local.get $f))) + (return) + ) ;; on_baz + (drop) + (return (i32.const 3)) + ) + + (func $inner-baz (result i32) + (call $handle_baz (ref.func $do_foo))) + (func (export "compose-handle-foo-baz") (result i32) + (call $handle_foo (ref.func $inner-baz))) + + (func $inner-foo (result i32) + (call $handle_foo (ref.func $do_foo))) + (func (export "compose-handle-baz-foo") (result i32) + (call $handle_baz (ref.func $inner-foo))) + (elem declare func $do_foo $inner-baz $inner-foo) +) +(register "baz") +(assert_return (invoke "compose-handle-baz-foo") (i32.const 1)) +(assert_return (invoke "compose-handle-foo-baz") (i32.const 1)) + +(module $quux + (type $ft (func (result i32))) + (type $ct (cont $ft)) + + (type $ft-2 (func (param i32) (result i32))) + (type $ct-2 (cont $ft-2)) + + (func $handle_foo (import "foo" "handle_foo") (param (ref $ft)) (result i32)) + (tag $foo (import "foo" "foo") (result i32)) + + (func $do_foo (result i32) + (suspend $foo)) + + (func $my_handle_foo (param $f (ref $ft)) (result i32) + (block $on_foo (result (ref $ct-2)) + (resume $ct (on $foo $on_foo) (cont.new $ct (local.get $f))) + (return) + ) ;; on_foo + (drop) + (return (i32.const 4)) + ) + + (func $inner-my-foo (result i32) + (call $my_handle_foo (ref.func $do_foo))) + (func (export "compose-handle-foo-my-foo") (result i32) + (call $handle_foo (ref.func $inner-my-foo))) + + (func $inner-foo (result i32) + (call $handle_foo (ref.func $do_foo))) + (func (export "compose-handle-my-foo-foo") (result i32) + (call $my_handle_foo (ref.func $inner-foo))) + (elem declare func $do_foo $inner-my-foo $inner-foo) +) +(register "quux") +(assert_return (invoke "compose-handle-foo-my-foo") (i32.const 4)) +(assert_return (invoke "compose-handle-my-foo-foo") (i32.const 1)) + +(module $bar + (type $ft (func (result i32))) + (type $ct (cont $ft)) + + (type $ft-2 (func (param i32) (result i32))) + (type $ct-2 (cont $ft-2)) + + (tag $foo (import "foo" "foo") (result i32)) + (tag $bar (result i32)) + (func $do_foo (result i32) + (suspend $foo)) + + ;; Don't handle the imported foo. + (func (export "skip-imported-foo") (result i32) + (block $on_bar (result (ref $ct-2)) + (resume $ct (on $bar $on_bar) (cont.new $ct (ref.func $do_foo))) + (unreachable) + ) + (unreachable)) + + ;; Handle the imported foo. + (func (export "handle-imported-foo") (result i32) + (block $on_foo (result (ref $ct-2)) + (resume $ct (on $foo $on_foo) (cont.new $ct (ref.func $do_foo))) + (unreachable) + ) + (drop) + (return (i32.const 2)) + ) + + (elem declare func $do_foo) +) +(register "bar") +(assert_return (invoke "handle-imported-foo") (i32.const 2)) +;; Due to issue #253, we need to make sure that nothing happens afterwards in +;; the test: +(assert_suspension (invoke "skip-imported-foo") "unhandled") diff --git a/tests/misc_testsuite/stack-switching/switch1.wast b/tests/misc_testsuite/stack-switching/switch1.wast new file mode 100644 index 000000000000..5a4047482627 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/switch1.wast @@ -0,0 +1,35 @@ +;;! stack_switching = true + +;; smoke test for switching: Only a single switch to a cotinuation created with +;; cont.new. +(module + + (type $ft0 (func)) + (type $ct0 (cont $ft0)) + + (type $ft1 (func (param (ref $ct0)))) + (type $ct1 (cont $ft1)) + + (func $print (import "spectest" "print_i32") (param i32)) + (tag $t) + + + (func $f + (cont.new $ct1 (ref.func $g)) + (switch $ct1 $t) + ) + (elem declare func $f) + + (func $g (type $ft1) + (call $print (i32.const 123)) + ) + (elem declare func $g) + + (func $entry (export "entry") (result i32) + (cont.new $ct0 (ref.func $f)) + (resume $ct0 (on $t switch)) + (i32.const 0) + ) +) + +(assert_return (invoke "entry" ) (i32.const 0)) diff --git a/tests/misc_testsuite/stack-switching/switch2.wast b/tests/misc_testsuite/stack-switching/switch2.wast new file mode 100644 index 000000000000..2bf84718fcf1 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/switch2.wast @@ -0,0 +1,71 @@ +;;! stack_switching = true +;;! gc = true + +;; switch to continuation created by switch +(module + + (rec + (type $ft (func (param i32 (ref null $ct)) (result i32))) + (type $ct (cont $ft))) + + (tag $t (result i32)) + + (func $f0 (type $ft) + ;; Just a wrapper around $f1 to make sure that the involved continuation + ;; chains consist of more than one element. + + (local.get 0) + (local.get 1) + (cont.new $ct (ref.func $f1)) + (resume $ct) + ) + (elem declare func $f0) + + (func $f1 (type $ft) + ;; add 1 to argument and pass to $g0 on switch + (local.get 0) + (i32.const 1) + (i32.add) + ;; prepare continuation + (cont.new $ct (ref.func $g0)) + (switch $ct $t) + (drop) ;; we won't run $g to completion + + ;; add 1 to payload received from $g1 + (i32.const 1) + (i32.add) + ) + (elem declare func $f1) + + (func $g0 (type $ft) + ;; Just a wrapper around $g1 to make sure that the involved continuation + ;; chains consist of more than one element. + + (local.get 0) + (local.get 1) + (cont.new $ct (ref.func $g1)) + (resume $ct) + ) + (elem declare func $g0) + + (func $g1 (type $ft) + ;; add 1 to argument received from $f1 + (local.get 0) + (i32.const 1) + (i32.add) + (local.get 1) + (switch $ct $t) + + ;; $f never switches back to us + (unreachable) + ) + (elem declare func $g1) + + (func $entry (export "entry") (result i32) + (i32.const 100) + (ref.null $ct) + (cont.new $ct (ref.func $f0)) + (resume $ct (on $t switch)) + ) +) +(assert_return (invoke "entry" ) (i32.const 103)) diff --git a/tests/misc_testsuite/stack-switching/switch3.wast b/tests/misc_testsuite/stack-switching/switch3.wast new file mode 100644 index 000000000000..58df4aa337b1 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/switch3.wast @@ -0,0 +1,95 @@ +;;! stack_switching = true +;;! gc = true + +;; resume continuation created by switch +(module + + (rec + (type $ft0 (func (param i32 (ref null $ct0)) (result i32))) + (type $ct0 (cont $ft0))) + + (type $ft1 (func (param i32) (result i32))) + (type $ct1 (cont $ft1)) + + (tag $t_switch (result i32)) + (tag $t_suspend (param i32) (result i32)) + + (func $f0 (type $ft0) + ;; Just a wrapper around $f1 to make sure that the involved continuation + ;; chains consist of more than one element. + + (local.get 0) + (local.get 1) + (cont.new $ct0 (ref.func $f1)) + (resume $ct0) + ) + (elem declare func $f0) + + (func $f1 (type $ft0) + ;; add 1 to argument and pass to $g on switch + (local.get 0) + (i32.const 1) + (i32.add) + ;; prepare continuation + (cont.new $ct0 (ref.func $g0)) + ;; switch to $g0 + (switch $ct0 $t_switch) + ;; g1 resumed us, installed suspend handler for t_suspend) + ;; drop null continuation and increment argument. + (drop) + (i32.const 1) + (i32.add) + (suspend $t_suspend) + + ;; add 1 to tag return value + (i32.const 1) + (i32.add) + ) + (elem declare func $f1) + + (func $g0 (type $ft0) + ;; Just a wrapper around $g1 to make sure that the involved continuation + ;; chains consist of more than one element. + + (local.get 0) + (local.get 1) + (cont.new $ct0 (ref.func $g1)) + (resume $ct0) + ) + (elem declare func $g0) + + (func $g1 (type $ft0) + (local $c (ref $ct1)) + + (block $handler (result i32 (ref $ct1)) + ;; add 1 to argument received from f1 on switch + (local.get 0) + (i32.const 1) + (i32.add) + (ref.null $ct0) ;; passed as payload + (local.get 1) ;; resumed + (resume $ct0 (on $t_suspend $handler)) + (unreachable) ;; f1 will suspend after the switch + ) + ;; stash continuation created by suspend in $f1 aside + (local.set $c) + ;; increment value received from suspend in $f1 + (i32.const 1) + (i32.add) + ;; ... and pass back to $f1 + (local.get $c) + (resume $ct1) + ;; increment $f1's return value + (i32.const 1) + (i32.add) + ) + (elem declare func $g1) + + (func $entry (export "entry") (result i32) + (i32.const 100) + (ref.null $ct0) + (cont.new $ct0 (ref.func $f0)) + (resume $ct0 (on $t_switch switch)) + ) +) +(assert_return (invoke "entry" ) (i32.const 106)) diff --git a/tests/misc_testsuite/stack-switching/switch4.wast b/tests/misc_testsuite/stack-switching/switch4.wast new file mode 100644 index 000000000000..4cf1d6b51bd9 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/switch4.wast @@ -0,0 +1,52 @@ +;;! stack_switching = true +;;! gc = true + +;; use cont.bind on continuation created by switch +(module + + (rec + (type $ft0 (func (param (ref null $ct1)) (result i32))) + (type $ct0 (cont $ft0)) + (type $ft1 (func (param i32 (ref null $ct1)) (result i32))) + (type $ct1 (cont $ft1))) + + (tag $t (result i32)) + + (func $f (type $ft1) + ;; add 1 to argument and pass to $g on switch + (local.get 0) + (i32.const 1) + (i32.add) + ;; prepare continuation + (cont.new $ct1 (ref.func $g)) + (cont.bind $ct1 $ct0) + (switch $ct0 $t) + (drop) ;; we won't run $g to completion + + ;; add 1 to payload received from $g + (i32.const 1) + (i32.add) + ) + (elem declare func $f) + + (func $g (type $ft1) + ;; add 1 to argument received from $f + (local.get 0) + (i32.const 1) + (i32.add) + (local.get 1) + (cont.bind $ct1 $ct0) + (switch $ct0 $t) + ;; $f never switches back to us + (unreachable) + ) + (elem declare func $g) + + (func $entry (export "entry") (result i32) + (i32.const 100) + (ref.null $ct1) + (cont.new $ct1 (ref.func $f)) + (resume $ct1 (on $t switch)) + ) +) +(assert_return (invoke "entry" ) (i32.const 103)) diff --git a/tests/misc_testsuite/stack-switching/switch5.wast b/tests/misc_testsuite/stack-switching/switch5.wast new file mode 100644 index 000000000000..207b0cef0533 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/switch5.wast @@ -0,0 +1,54 @@ +;;! stack_switching = true + +;; switch to continuation created by suspend +(module + + (type $ft0 (func (result i32))) + (type $ct0 (cont $ft0)) + + (type $ft1 (func (result i32))) + (type $ct1 (cont $ft1)) + + (type $ft2 (func (param i32 (ref $ct0)) (result i32))) + (type $ct2 (cont $ft2)) + + (tag $t_suspend (result i32 (ref $ct0))) + (tag $t_switch (result i32)) + + (global $c (mut (ref null $ct2)) (ref.null $ct2)) + + + (func $f (result i32) + (suspend $t_suspend) + (drop) ;; drops continuation created by switch without running to completion + + ;; We increment the switch payload and return it to our handler + (i32.const 1) + (i32.add) + ) + (elem declare func $f) + + (func $g (result i32) + (i32.const 100) + (global.get $c) + (switch $ct2 $t_switch) + ;; we never switch back here + (unreachable) + ) + + (elem declare func $g) + + (func $entry (export "entry") (result i32) + (block $handler (result (ref $ct2)) + (cont.new $ct0 (ref.func $f)) + (resume $ct0 (on $t_suspend $handler)) + (unreachable) + ) + (global.set $c) + + (cont.new $ct1 (ref.func $g)) + (resume $ct1 (on $t_switch switch)) + + ) +) +(assert_return (invoke "entry" ) (i32.const 101)) diff --git a/tests/misc_testsuite/stack-switching/switch6.wast b/tests/misc_testsuite/stack-switching/switch6.wast new file mode 100644 index 000000000000..c634adcae9b4 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/switch6.wast @@ -0,0 +1,37 @@ +;;! stack_switching = true + +;; suspend past a switch handler for same tag +(module + + (type $ft (func)) + (type $ct (cont $ft)) + + (tag $t) + + (func $f + (cont.new $ct (ref.func $g)) + (resume $ct (on $t switch)) + ;; $g will suspend and we will not come back here + (unreachable) + ) + (elem declare func $f) + + (func $g (type $ft) + (suspend $t) + ;; we will not come back here + (unreachable) + ) + (elem declare func $g) + + (func $entry (export "entry") (result i32) + (block $handler (result (ref $ct)) + (cont.new $ct (ref.func $f)) + (resume $ct (on $t switch) (on $t $handler)) + ;; we will have a suspension + (unreachable) + ) + (drop) + (i32.const 100) + ) +) +(assert_return (invoke "entry" ) (i32.const 100)) diff --git a/tests/misc_testsuite/stack-switching/switch7.wast b/tests/misc_testsuite/stack-switching/switch7.wast new file mode 100644 index 000000000000..1520286597b0 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/switch7.wast @@ -0,0 +1,52 @@ +;;! stack_switching = true + +;; switch past a suspend handler for same tag +(module + + (type $ft0 (func (result i32))) + (type $ct0 (cont $ft0)) + + (type $ft1 (func (param i32) (result i32))) + (type $ct1 (cont $ft1)) + + (type $ft2 (func (param (ref $ct0)) (result i32))) + (type $ct2 (cont $ft2)) + + (tag $t (result i32)) + + (func $f (result i32) + (block $handler (result (ref $ct1)) + (cont.new $ct0 (ref.func $g)) + (resume $ct0 (on $t $handler)) + ;; $g will switch, we won't come back here + (unreachable) + ) + ;; we will not suspend + (unreachable) + ) + (elem declare func $f) + + (func $g (result i32) + (cont.new $ct2 (ref.func $h)) + (switch $ct2 $t) + ;; we won't come back here + (unreachable) + ) + (elem declare func $g) + + (func $h (type $ft2) + (i32.const 100) + ) + (elem declare func $h) + + (func $entry (export "entry") (result i32) + (block $handler (result (ref $ct1)) + (cont.new $ct0 (ref.func $f)) + (resume $ct0 (on $t switch) (on $t $handler)) + (return) + ) + ;; we will not suspend + (unreachable) + ) +) +(assert_return (invoke "entry" ) (i32.const 100)) diff --git a/tests/misc_testsuite/stack-switching/switch8.wast b/tests/misc_testsuite/stack-switching/switch8.wast new file mode 100644 index 000000000000..b92ab8b25352 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/switch8.wast @@ -0,0 +1,36 @@ +;;! stack_switching = true + +;; try to switch to an already consumed continuation +(module + (type $ft0 (func)) + (type $ct0 (cont $ft0)) + + (type $ft1 (func (param (ref null $ct0)))) + (type $ct1 (cont $ft1)) + + (func $print (import "spectest" "print_i32") (param i32)) + (tag $t) + + + (func $f + (local $c (ref $ct1)) + (ref.null $ct0) ;; argument to $g + (cont.new $ct1 (ref.func $g)) + (local.tee $c) + (resume $ct1) + + ;; this should fail, we already used the continuation + (local.get $c) + (switch $ct1 $t) + ) + (elem declare func $f) + + (func $g (type $ft1)) + (elem declare func $g) + + (func $entry (export "entry") + (cont.new $ct0 (ref.func $f)) + (resume $ct0 (on $t switch)) + ) +) +(assert_trap (invoke "entry") "continuation already consumed") diff --git a/tests/misc_testsuite/stack-switching/switch_null.wast b/tests/misc_testsuite/stack-switching/switch_null.wast new file mode 100644 index 000000000000..e03097ccb4c7 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/switch_null.wast @@ -0,0 +1,16 @@ +;;! stack_switching = true +(module + (type $ft0 (func)) + (type $ct0 (cont $ft0)) + + (type $ft1 (func (param (ref $ct0)))) + (type $ct1 (cont $ft1)) + + (tag $t) + + (func $entry (export "entry") + (switch $ct1 $t (ref.null $ct1)) + ) +) + +(assert_trap (invoke "entry") "null reference") diff --git a/tests/misc_testsuite/stack-switching/unhandled.wast b/tests/misc_testsuite/stack-switching/unhandled.wast new file mode 100644 index 000000000000..dc30c71ec4d4 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/unhandled.wast @@ -0,0 +1,24 @@ +;;! stack_switching = true +;; Test unhandled suspension + +(module + (type $ft (func)) + (type $ct (cont $ft)) + (tag $t) + + (func $suspend + (suspend $t)) + (elem declare func $suspend) + + (func $unhandled-0 (export "unhandled-0") + (call $suspend)) + + (func $unhandled-1 (export "unhandled-1") + (resume $ct (cont.new $ct (ref.func $suspend)))) +) + +;; TODO(dhil): Suspending on the main thread currently causes an +;; unrecoverable panic. Instead we should emit the UnhandledTrap trap +;; code; once this has been implemented the below test should pass. +;;(assert_suspension (invoke "unhandled-0") "unhandled") +(assert_suspension (invoke "unhandled-1") "unhandled") \ No newline at end of file From dd63ee877b9b6bc04dc7476bbd5c1f125ef6221f Mon Sep 17 00:00:00 2001 From: Frank Emrich Date: Mon, 21 Apr 2025 22:24:00 +0100 Subject: [PATCH 6/6] prtest:full