diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index b3104ffaa77a..4c8aab1584a6 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -550,6 +550,8 @@ pub(crate) fn emit( in_payload0, out_payload0, } => { + let stack_map = state.take_stack_map(); + // Note that we do not emit anything for preserving and restoring // ordinary registers here: That's taken care of by regalloc for us, // since we marked this instruction as clobbering all registers. @@ -643,6 +645,9 @@ pub(crate) fn emit( asm::inst::jmpq_m::new(tmp1.to_reg()).emit(sink, info, state); sink.bind_label(resume, state.ctrl_plane_mut()); + if let Some(s) = stack_map { + sink.push_user_stack_map(state, sink.cur_offset(), s); + } } Inst::JmpKnown { dst } => uncond_jmp(sink, *dst), diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 49c0be93905b..38d1bbf88f01 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -1469,7 +1469,9 @@ impl MachInst for Inst { fn is_safepoint(&self) -> bool { match self { - Inst::CallKnown { .. } | Inst::CallUnknown { .. } => true, + Inst::CallKnown { .. } | Inst::CallUnknown { .. } | Inst::StackSwitchBasic { .. } => { + true + } _ => false, } } diff --git a/cranelift/filetests/filetests/isa/x64/stack_switch_stack_map.clif b/cranelift/filetests/filetests/isa/x64/stack_switch_stack_map.clif new file mode 100644 index 000000000000..dff66b81370e --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/stack_switch_stack_map.clif @@ -0,0 +1,14 @@ +test compile +set opt_level=speed +set stack_switch_model=basic +target x86_64 + +;; A stack switch is a GC safepoint. Its stack map is associated with the +;; resume PC saved in the outgoing control context. +function %switch_with_stack_map(i64, i64, i64) -> i64 { + ss0 = explicit_slot 4 + +block0(v0: i64, v1: i64, v2: i64): + v3 = stack_switch v0, v1, v2, stack_map=[i32 @ ss0+0] + return v3 +} diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 71b07ef66604..1f4ac0e58f26 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -46,6 +46,20 @@ use wasmtime_environ::{ }; use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK}; +/// Function-local stack slots backing a continuation's +/// `VMPayloads::values`. +/// +/// The values and their GC-reference markers are logically one +/// payload descriptor but use distinct stack slots so that Cranelift +/// can assign them distinct alias regions. The marker slot is only +/// allocated when this function contains a stack switching site whose +/// payloads may contain GC references. +#[derive(Clone, Copy)] +pub(crate) struct VMPayloadStackSlots { + pub(crate) values: ir::StackSlot, + pub(crate) gc_ref_markers: Option, +} + #[derive(Copy, Clone, Debug)] pub(crate) enum Extension { Sign, @@ -216,10 +230,9 @@ pub struct FuncEnvironment<'module_environment> { /// current stack's `handler_list` field. stack_switching_handler_list_buffer: Option, - /// Used by the stack switching feature. If set, we have a allocated a - /// slot on this function's stack to be used for the - /// current continuation's `values` field. - stack_switching_values_buffer: Option, + /// Used by the stack switching feature. If set, these are the stack slots + /// backing the current continuation's `values` field. + stack_switching_values_storage: Option, /// The stack-slot used for exposing Wasm state via debug /// instrumentation, if any, and the builder containing its metadata. @@ -300,7 +313,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { stack_limit_at_function_entry: None, stack_switching_handler_list_buffer: None, - stack_switching_values_buffer: None, + stack_switching_values_storage: None, state_slot: None, next_srcloc: ir::SourceLoc::default(), @@ -5259,8 +5272,9 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder<'_>, contobj: ir::Value, args: &[ir::Value], + arg_types: &[WasmValType], ) -> ir::Value { - stack_switching::instructions::translate_cont_bind(self, builder, contobj, args) + stack_switching::instructions::translate_cont_bind(self, builder, contobj, args, arg_types) } pub fn translate_cont_new( @@ -5340,13 +5354,15 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder<'_>, tag_index: u32, suspend_args: &[ir::Value], - tag_return_types: &[ir::Type], + suspend_arg_types: &[WasmValType], + tag_return_types: &[WasmValType], ) -> WasmResult> { stack_switching::instructions::translate_suspend( self, builder, tag_index, suspend_args, + suspend_arg_types, tag_return_types, ) } @@ -5358,7 +5374,8 @@ impl FuncEnvironment<'_> { tag_index: u32, contobj: ir::Value, switch_args: &[ir::Value], - return_types: &[ir::Type], + switch_arg_types: &[WasmValType], + return_types: &[WasmValType], ) -> WasmResult> { stack_switching::instructions::translate_switch( self, @@ -5366,6 +5383,7 @@ impl FuncEnvironment<'_> { tag_index, contobj, switch_args, + switch_arg_types, return_types, ) } diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs index b88d21c050d0..e5094f79de48 100644 --- a/crates/cranelift/src/func_environ/gc.rs +++ b/crates/cranelift/src/func_environ/gc.rs @@ -2,7 +2,7 @@ use crate::TRAP_ARRAY_OUT_OF_BOUNDS; use crate::bounds_checks::BoundsCheck; -use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment}; +use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment, stack_switching::fatpointer}; use crate::translate::{ Heap, HeapData, MemoryKind, StructFieldsVec, TargetEnvironment, VmctxLoadChain, }; @@ -377,10 +377,7 @@ pub fn read_field_at_addr( .call(get_interned_func_ref, &[vmctx, func_ref_id, expected_ty]); builder.func.dfg.first_result(call_inst) } - WasmHeapTopType::Cont => { - // TODO(#10248) GC integration for stack switching - return stack_switching_unsupported(); - } + WasmHeapTopType::Cont => read_cont_ref_at_addr(func_env, builder, addr, flags)?, }, }, }; @@ -431,6 +428,89 @@ pub fn intern_func_ref( Ok(builder.ins().ireduce(ir::types::I32, func_ref_id)) } +fn intern_cont_ref( + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder<'_>, + ref_type: WasmRefType, + contobj: ir::Value, +) -> ir::Value { + assert_eq!(ref_type.heap_type.top(), WasmHeapTopType::Cont); + + if ref_type.heap_type == WasmHeapType::NoCont { + let zero = builder.ins().iconst(ir::types::I32, 0); + + if !ref_type.nullable { + builder.ins().trapz(zero, TRAP_INTERNAL_ASSERT); + } + + return zero; + } + + let (revision, contref) = fatpointer::deconstruct(func_env, &mut builder.cursor(), contobj); + let vmctx = func_env.vmctx_val(&mut builder.cursor()); + let intern = func_env + .builtin_functions + .intern_contref_for_gc_heap(builder.func); + let call = builder.ins().call(intern, &[vmctx, contref, revision]); + let id = builder.func.dfg.first_result(call); + builder.ins().ireduce(ir::types::I32, id) +} + +fn read_cont_ref_at_addr( + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder<'_>, + addr: ir::Value, + flags: ir::MemFlagsData, +) -> WasmResult { + let id = builder.ins().load(ir::types::I32, flags, addr, 0); + + // We create a stack slot to hold the continuation values (16 + // bytes), and pass the address of this slot as the out parameter + // to the builtin. + let pointer_type = func_env.pointer_type(); + let pointer_bytes = pointer_type.bytes(); + let slot = builder.create_sized_stack_slot(ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + 2 * pointer_bytes, + u8::try_from(pointer_bytes.trailing_zeros()).unwrap(), + )); + let out_result = builder.ins().stack_addr(pointer_type, slot, 0); + + let vmctx = func_env.vmctx_val(&mut builder.cursor()); + let get = func_env + .builtin_functions + .get_interned_contref(builder.func); + builder.ins().call(get, &[vmctx, id, out_result]); + + let region = func_env.alias_regions.stack_slot_region(builder.func, slot); + let flags = ir::MemFlagsData::trusted().with_alias_region(Some(region)); + let contref = builder.ins().load(pointer_type, flags, out_result, 0); + let revision = builder.ins().load( + pointer_type, + flags, + out_result, + i32::try_from(pointer_bytes).unwrap(), + ); + Ok(fatpointer::construct( + func_env, + &mut builder.cursor(), + revision, + contref, + )) +} + +fn write_cont_ref_at_addr( + func_env: &mut FuncEnvironment<'_>, + builder: &mut FunctionBuilder<'_>, + ref_type: WasmRefType, + flags: ir::MemFlagsData, + field_addr: ir::Value, + contobj: ir::Value, +) { + let id = intern_cont_ref(func_env, builder, ref_type, contobj); + builder.ins().store(flags, id, field_addr, 0); +} + fn write_func_ref_at_addr( func_env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, @@ -487,7 +567,9 @@ pub fn write_field_at_addr( func_env, builder, r, field_addr, new_val, flags, )?; } - WasmHeapTopType::Cont => return stack_switching_unsupported(), + WasmHeapTopType::Cont => { + write_cont_ref_at_addr(func_env, builder, r, flags, field_addr, new_val) + } }, WasmStorageType::Val(_) => { assert_eq!( @@ -527,6 +609,10 @@ fn default_value( } WasmValType::Ref(r) => { assert!(r.nullable); + if r.heap_type.top() == WasmHeapTopType::Cont { + let zero = cursor.ins().iconst(func_env.pointer_type(), 0); + return fatpointer::construct(func_env, cursor, zero, zero); + } let (ty, needs_stack_map) = func_env.reference_type(r.heap_type); // NB: The collector doesn't need to know about null references. diff --git a/crates/cranelift/src/func_environ/gc/copying.rs b/crates/cranelift/src/func_environ/gc/copying.rs index 7f3a56541e67..ca1ec4a43b09 100644 --- a/crates/cranelift/src/func_environ/gc/copying.rs +++ b/crates/cranelift/src/func_environ/gc/copying.rs @@ -440,7 +440,9 @@ impl GcCompiler for CopyingCompiler { // store the reference directly. unbarriered_store_gc_ref(builder, r.heap_type, field_addr, val, flags)?; } - WasmHeapTopType::Cont => return super::stack_switching_unsupported(), + WasmHeapTopType::Cont => { + write_field_at_addr(func_env, builder, ty, field_addr, val)? + } }, WasmStorageType::I8 => { assert_eq!(builder.func.dfg.value_type(val), ir::types::I32); diff --git a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs index 05b92e7779de..89cc0083b8ef 100644 --- a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs +++ b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs @@ -34,7 +34,7 @@ pub(crate) fn deconstruct<'a>( /// Constructs a continuation object from a given contref and revision pointer. /// The contref_addr may be 0, to indicate that we want to build a wasm null reference. pub(crate) fn construct<'a>( - env: &mut crate::func_environ::FuncEnvironment<'a>, + env: &crate::func_environ::FuncEnvironment<'a>, pos: &mut cranelift_codegen::cursor::FuncCursor, revision_counter: ir::Value, contref_addr: ir::Value, diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs index 56386c797c21..dfd7282b8d6e 100644 --- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs +++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs @@ -8,7 +8,10 @@ use cranelift_codegen::ir::{self, MemFlagsData}; use cranelift_codegen::ir::{Block, BlockCall, InstBuilder, JumpTableData}; use cranelift_frontend::FunctionBuilder; use itertools::{Either, Itertools}; -use wasmtime_environ::{PtrSize, TagIndex, TypeIndex, WasmResult, WasmValType, wasm_unsupported}; +use wasmtime_environ::{ + PtrSize, TagIndex, TypeIndex, WasmHeapType, WasmRefType, WasmResult, WasmValType, + wasm_unsupported, +}; fn control_context_size(triple: &target_lexicon::Triple) -> WasmResult { match (triple.architecture, triple.operating_system) { @@ -31,7 +34,7 @@ pub(crate) mod stack_switching_helpers { use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{StackSlot, StackSlotKind::*}; use cranelift_frontend::FunctionBuilder; - use wasmtime_environ::PtrSize; + use wasmtime_environ::{PtrSize, WasmValType}; /// Provides information about the layout of a type when it is used as an /// element in a host array. This is used for `VMHostArrayRef`. @@ -69,7 +72,17 @@ pub(crate) mod stack_switching_helpers { phantom: PhantomData, } - pub type VMPayloads = VMHostArrayRef; + /// Compile-time reference to a runtime `VMPayloads` descriptor. + /// + /// This wrapper keeps the payload buffer and its GC-reference markers + /// paired. Its core invariant is that `gc_ref_data`, when non-null, points + /// to one marker byte for every slot in `buffer`; callers must update a + /// slot's value and marker together. + #[derive(Copy, Clone)] + pub struct VMPayloads { + address: ir::Value, + buffer: VMHostArrayRef, + } // Actually a vector of *mut VMTagDefinition pub type VMHandlerList = VMHostArrayRef<*mut u8>; @@ -105,7 +118,7 @@ pub(crate) mod stack_switching_helpers { ) -> VMPayloads { let offset: i64 = env.offsets.ptr.vmcontref_args().into(); let address = builder.ins().iadd_imm_s(self.address, offset); - VMPayloads::new(address) + VMPayloads::new(env, address) } pub fn values<'a>( @@ -115,7 +128,7 @@ pub(crate) mod stack_switching_helpers { ) -> VMPayloads { let offset: i64 = env.offsets.ptr.vmcontref_values().into(); let address = builder.ins().iadd_imm_s(self.address, offset); - VMPayloads::new(address) + VMPayloads::new(env, address) } pub fn common_stack_information<'a>( @@ -234,6 +247,148 @@ pub(crate) mod stack_switching_helpers { } } + impl VMPayloads { + pub fn new(env: &mut crate::func_environ::FuncEnvironment<'_>, address: ir::Value) -> Self { + debug_assert_eq!(env.offsets.ptr.vmpayloads_buffer(), 0); + Self { + address, + buffer: VMHostArrayRef::new(address), + } + } + + pub fn get_gc_ref_data( + &self, + env: &mut crate::func_environ::FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let offset: i32 = env.offsets.ptr.vmpayloads_gc_ref_data().into(); + let region = env.alias_regions.vmcontref_region(builder.func); + let flags = ir::MemFlagsData::trusted().with_alias_region(Some(region)); + builder + .ins() + .load(env.pointer_type(), flags, self.address, offset) + } + + pub fn set_gc_ref_data( + &self, + env: &mut crate::func_environ::FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + data: ir::Value, + ) { + let offset: i32 = env.offsets.ptr.vmpayloads_gc_ref_data().into(); + let region = env.alias_regions.vmcontref_region(builder.func); + let flags = ir::MemFlagsData::trusted().with_alias_region(Some(region)); + builder.ins().store(flags, data, self.address, offset); + } + + /// Prepares stack storage for this payload buffer. + /// + /// Payload values and their GC-reference markers use distinct stack + /// slots so that they have distinct alias regions. The descriptor's + /// capacity and pointers always describe the storage selected at this + /// particular instruction site. Both slots retain their identities and + /// grow in place as later sites require more capacity. + pub fn prepare_stack_storage( + &self, + env: &mut crate::func_environ::FuncEnvironment<'_>, + builder: &mut FunctionBuilder, + capacity: u32, + initial_types: &[WasmValType], + needs_gc_ref_markers: bool, + existing_slots: Option, + ) -> crate::func_environ::VMPayloadStackSlots { + debug_assert!(capacity > 0); + debug_assert!(initial_types.len() <= usize::try_from(capacity).unwrap()); + + let (align, entry_size) = + ::vmhostarray_entry_layout(&env.offsets.ptr); + let values_size = capacity + .checked_mul(entry_size) + .expect("stack switching values buffer size should fit in u32"); + + let values_slot = match existing_slots.map(|slots| slots.values) { + Some(slot) => { + // Keep the slot identity stable: `stack_addr` instructions + // emitted at earlier sites continue to refer to this slot + // when a later site requires more storage. + let slot_data = &mut builder.func.sized_stack_slots[slot]; + debug_assert!(align <= slot_data.align_shift); + debug_assert_eq!(slot_data.kind, ExplicitSlot); + slot_data.size = slot_data.size.max(values_size); + slot + } + None => builder.create_sized_stack_slot(ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + values_size, + align, + )), + }; + + let capacity_value = builder.ins().iconst(I32, i64::from(capacity)); + let values_data = builder.ins().stack_addr(env.pointer_type(), values_slot, 0); + self.set_capacity(env, builder, capacity_value); + self.set_data(env, builder, values_data); + + let mut gc_ref_markers = existing_slots.and_then(|slots| slots.gc_ref_markers); + if needs_gc_ref_markers { + let marker_slot = match gc_ref_markers { + Some(slot) => { + let slot_data = &mut builder.func.sized_stack_slots[slot]; + debug_assert_eq!(slot_data.kind, ExplicitSlot); + slot_data.size = slot_data.size.max(capacity); + slot + } + None => builder.create_sized_stack_slot(ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + capacity, + 0, + )), + }; + gc_ref_markers = Some(marker_slot); + + let marker_data = builder.ins().stack_addr(env.pointer_type(), marker_slot, 0); + self.set_gc_ref_data(env, builder, marker_data); + + let region = env + .alias_regions + .stack_slot_region(builder.func, marker_slot); + let flags = ir::MemFlagsData::trusted().with_alias_region(Some(region)); + let zero = builder.ins().iconst(I8, 0); + for offset in 0..capacity { + builder + .ins() + .store(flags, zero, marker_data, i32::try_from(offset).unwrap()); + } + for (offset, ty) in initial_types.iter().enumerate() { + if ty.is_vmgcref_type_and_not_i31() { + let marker = builder + .ins() + .iconst(I8, i64::from(wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF)); + builder.ins().store( + flags, + marker, + marker_data, + i32::try_from(offset).unwrap(), + ); + } + } + } + + crate::func_environ::VMPayloadStackSlots { + values: values_slot, + gc_ref_markers, + } + } + } + + impl core::ops::Deref for VMPayloads { + type Target = VMHostArrayRef; + + fn deref(&self) -> &Self::Target { + &self.buffer + } + } + impl VMHostArrayRef { pub(crate) fn new(address: ir::Value) -> Self { Self { @@ -333,7 +488,7 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, arg_count: i32, - ) -> ir::Value { + ) -> (ir::Value, ir::Value) { let data = self.get_data(env, builder); let original_length = self.get_length(env, builder); let new_length = builder @@ -346,7 +501,7 @@ pub(crate) mod stack_switching_helpers { let byte_offset = builder .ins() .imul_imm_s(original_length, i64::from(entry_size)); - builder.ins().iadd(data, byte_offset) + (builder.ins().iadd(data, byte_offset), original_length) } pub fn allocate_or_reuse_stack_slot<'a>( @@ -393,13 +548,13 @@ pub(crate) mod stack_switching_helpers { } /// Loads n entries from this Vector object, where n is the length of - /// `load_types`, which also gives the types of the values to load. + /// `load_types`, which also gives the Wasm types of the values to load. /// Loading starts at index 0 of the Vector object. pub fn load_data_entries<'a>( &self, env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, - load_types: &[ir::Type], + load_types: &[WasmValType], ) -> Vec { let region = env .alias_regions @@ -410,10 +565,14 @@ pub(crate) mod stack_switching_helpers { let mut values = vec![]; let mut offset = 0; let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr); - for valtype in load_types { + for wasm_ty in load_types { + let valtype = crate::value_type(env.isa(), *wasm_ty); let val = builder .ins() - .load(*valtype, memflags, data_start_pointer, offset); + .load(valtype, memflags, data_start_pointer, offset); + if env.val_ty_needs_stack_map(*wasm_ty) { + builder.declare_value_needs_stack_map(val); + } values.push(val); offset += i32::try_from(entry_size).unwrap(); } @@ -934,6 +1093,10 @@ pub(crate) mod stack_switching_helpers { use helpers::VMStackChain; use stack_switching_helpers as helpers; +fn types_need_gc_ref_markers(types: &[WasmValType]) -> bool { + types.iter().any(|ty| ty.is_vmgcref_type_and_not_i31()) +} + /// Stores the given arguments in the appropriate `VMPayloads` object in the /// continuation. If the continuation was never invoked, use the `args` object. /// Otherwise, use the `values` object. @@ -941,8 +1104,11 @@ pub(crate) fn vmcontref_store_payloads<'a>( env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, values: &[ir::Value], + types: &[WasmValType], contref: ir::Value, ) { + debug_assert_eq!(values.len(), types.len()); + let needs_gc_ref_markers = types_need_gc_ref_markers(types); let count = i32::try_from(values.len()).expect("Number of stack switching payloads should fit in i32"); if values.len() > 0 { @@ -950,6 +1116,9 @@ pub(crate) fn vmcontref_store_payloads<'a>( let use_payloads_block = builder.create_block(); let store_data_block = builder.create_block(); builder.append_block_param(store_data_block, env.pointer_type()); + if needs_gc_ref_markers { + builder.append_block_param(store_data_block, env.pointer_type()); + } let co = helpers::VMContRef::new(contref); let csi = co.common_stack_information(env, builder); @@ -963,11 +1132,15 @@ pub(crate) fn vmcontref_store_payloads<'a>( builder.seal_block(use_args_block); let args = co.args(env, builder); - let ptr = args.occupy_next_slots(env, builder, count); + let (ptr, original_length) = args.occupy_next_slots(env, builder, count); + let mut block_args = vec![BlockArg::Value(ptr)]; + if needs_gc_ref_markers { + let gc_refs_ptr = args.get_gc_ref_data(env, builder); + let gc_refs_ptr = builder.ins().iadd(gc_refs_ptr, original_length); + block_args.push(BlockArg::Value(gc_refs_ptr)); + } - builder - .ins() - .jump(store_data_block, &[BlockArg::Value(ptr)]); + builder.ins().jump(store_data_block, &block_args); } { @@ -978,10 +1151,14 @@ pub(crate) fn vmcontref_store_payloads<'a>( // This also checks that the buffer is large enough to hold // `values.len()` more elements. - let ptr = payloads.occupy_next_slots(env, builder, count); - builder - .ins() - .jump(store_data_block, &[BlockArg::Value(ptr)]); + let (ptr, original_length) = payloads.occupy_next_slots(env, builder, count); + let mut block_args = vec![BlockArg::Value(ptr)]; + if needs_gc_ref_markers { + let gc_refs_ptr = payloads.get_gc_ref_data(env, builder); + let gc_refs_ptr = builder.ins().iadd(gc_refs_ptr, original_length); + block_args.push(BlockArg::Value(gc_refs_ptr)); + } + builder.ins().jump(store_data_block, &block_args); } { @@ -989,6 +1166,8 @@ pub(crate) fn vmcontref_store_payloads<'a>( builder.seal_block(store_data_block); let ptr = builder.block_params(store_data_block)[0]; + let gc_refs_ptr = + needs_gc_ref_markers.then(|| builder.block_params(store_data_block)[1]); // Store the values. let region = env @@ -996,8 +1175,22 @@ pub(crate) fn vmcontref_store_payloads<'a>( .continuation_stack_memory_region(builder.func); let memflags = ir::MemFlagsData::trusted().with_alias_region(Some(region)); let mut offset = 0; - for value in values { + for (index, (value, ty)) in values.iter().zip(types).enumerate() { builder.ins().store(memflags, *value, ptr, offset); + if let Some(gc_refs_ptr) = gc_refs_ptr { + let marker = if ty.is_vmgcref_type_and_not_i31() { + i64::from(wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF) + } else { + 0 + }; + let marker = builder.ins().iconst(I8, marker); + builder.ins().store( + memflags, + marker, + gc_refs_ptr, + i32::try_from(index).unwrap(), + ); + } offset += i32::from(env.offsets.ptr.maximum_value_size()); } } @@ -1276,6 +1469,7 @@ pub(crate) fn translate_cont_bind<'a>( builder: &mut FunctionBuilder, contobj: ir::Value, args: &[ir::Value], + arg_types: &[WasmValType], ) -> ir::Value { let (witness, contref) = fatpointer::deconstruct(env, &mut builder.cursor(), contobj); @@ -1287,7 +1481,7 @@ pub(crate) fn translate_cont_bind<'a>( let evidence = builder.ins().icmp(IntCC::Equal, witness, revision); env.trapz(builder, evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); - vmcontref_store_payloads(env, builder, args, contref); + vmcontref_store_payloads(env, builder, args, arg_types, contref); let revision = vmcontref.incr_revision(env, builder, revision); let contobj = fatpointer::construct(env, &mut builder.cursor(), revision, contref); @@ -1310,12 +1504,16 @@ pub(crate) fn translate_cont_new<'a>( let nreturns = builder .ins() .iconst(I32, i64::try_from(return_types.len()).unwrap()); + let gc_refs = builder.ins().iconst( + I32, + i64::from(u8::from(types_need_gc_ref_markers(arg_types))), + ); let cont_new_func = env.builtin_functions.cont_new(&mut builder.func); let vmctx = env.vmctx_val(&mut builder.cursor()); let call_inst = builder .ins() - .call(cont_new_func, &[vmctx, func, nargs, nreturns]); + .call(cont_new_func, &[vmctx, func, nargs, nreturns, gc_refs]); let contref = *builder.func.dfg.inst_results(call_inst).first().unwrap(); let tag = helpers::VMContRef::new(contref).get_revision(env, builder); @@ -1399,6 +1597,10 @@ fn translate_resume_impl<'a>( resume_payload: ResumePayload<'_>, resumetable: &[(u32, Option)], ) -> WasmResult> { + let resume_arg_types = env + .continuation_arguments(TypeIndex::from_u32(type_index)) + .to_vec(); + // The resume instruction is the most involved instruction to // compile as it is responsible for both continuation application // and control tag dispatch. @@ -1480,10 +1682,17 @@ fn translate_resume_impl<'a>( match resume_payload { ResumePayload::Values(resume_args) => { + debug_assert_eq!(resume_args.len(), resume_arg_types.len()); let _next_revision = vmcontref.incr_revision(env, builder, revision); if resume_args.len() > 0 { // We store the arguments in the `VMContRef` to be resumed. - vmcontref_store_payloads(env, builder, resume_args, resume_contref); + vmcontref_store_payloads( + env, + builder, + resume_args, + &resume_arg_types, + resume_contref, + ); } } ResumePayload::Throw { .. } | ResumePayload::ThrowRef(_) => { @@ -1771,11 +1980,7 @@ fn translate_resume_impl<'a>( preamble_blocks.push(preamble_block); builder.switch_to_block(preamble_block); - let param_types = env.tag_params(TagIndex::from_u32(handle_tag)); - let param_types: Vec = param_types - .iter() - .map(|wty| crate::value_type(env.isa(), *wty)) - .collect(); + let param_types = env.tag_params(TagIndex::from_u32(handle_tag)).to_vec(); let values = suspended_contref.values(env, builder); let mut suspend_args: Vec = @@ -1840,11 +2045,9 @@ fn translate_resume_impl<'a>( returned_csi.set_state_returned(env, builder); // Load the values returned by the continuation. - let return_types: Vec<_> = env + let return_types = env .continuation_returns(TypeIndex::from_u32(type_index)) - .iter() - .map(|ty| crate::value_type(env.isa(), *ty)) - .collect(); + .to_vec(); let payloads = returned_contref.args(env, builder); let return_values = payloads.load_data_entries(env, builder, &return_types); payloads.clear(env, builder, true); @@ -1858,7 +2061,7 @@ fn load_resume_values_or_throw<'a>( builder: &mut FunctionBuilder, result: ControlEffect, continuation: helpers::VMContRef, - return_types: &[ir::Type], + return_types: &[WasmValType], ) -> WasmResult> { let throw_block = builder.create_block(); let values_block = builder.create_block(); @@ -1873,7 +2076,11 @@ fn load_resume_values_or_throw<'a>( let values = continuation.values(env, builder); // Exception references use Wasmtime's compressed, 32-bit GC-reference // representation. - let exnref = values.load_data_entries(env, builder, &[I32])[0]; + let exnref_ty = WasmValType::Ref(WasmRefType { + nullable: false, + heap_type: WasmHeapType::Exn, + }); + let exnref = values.load_data_entries(env, builder, &[exnref_ty])[0]; values.clear(env, builder, true); gc::translate_exn_throw_ref(env, builder, exnref)?; @@ -1890,8 +2097,10 @@ pub(crate) fn translate_suspend<'a>( builder: &mut FunctionBuilder, tag_index: u32, suspend_args: &[ir::Value], - tag_return_types: &[ir::Type], + suspend_arg_types: &[WasmValType], + tag_return_types: &[WasmValType], ) -> WasmResult> { + debug_assert_eq!(suspend_args.len(), suspend_arg_types.len()); let tag_addr = tag_address(env, builder, tag_index); let vmctx = env.vmctx_val(&mut builder.cursor()); @@ -1913,16 +2122,21 @@ pub(crate) fn translate_suspend<'a>( // return values including the possible exception reference. let values = active_contref.values(env, builder); let required_capacity = u32::try_from(std::cmp::max( - 1, // This account for the possible exception reference. + 1, // This accounts for the possible exception reference. std::cmp::max(suspend_args.len(), tag_return_types.len()), )) .expect("Number of stack switching payloads should fit in u32"); - env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot( + let needs_gc_ref_markers = + types_need_gc_ref_markers(suspend_arg_types) || types_need_gc_ref_markers(tag_return_types); + let existing_storage = env.stack_switching_values_storage; + env.stack_switching_values_storage = Some(values.prepare_stack_storage( env, builder, required_capacity, - env.stack_switching_values_buffer, + suspend_arg_types, + needs_gc_ref_markers, + existing_storage, )); if suspend_args.len() > 0 { @@ -1952,13 +2166,20 @@ pub(crate) fn translate_suspend<'a>( // A normal resume supplies the tag's return values. `resume_throw_ref` // supplies an exception reference and throws it at this suspension point. - load_resume_values_or_throw( + let return_values = load_resume_values_or_throw( env, builder, ControlEffect::from_u64(result), active_contref, tag_return_types, - ) + ); + + if needs_gc_ref_markers { + let zero = builder.ins().iconst(env.pointer_type(), 0); + values.set_gc_ref_data(env, builder, zero); + } + + return_values } pub(crate) fn translate_switch<'a>( @@ -1967,7 +2188,8 @@ pub(crate) fn translate_switch<'a>( tag_index: u32, switchee_contobj: ir::Value, switch_args: &[ir::Value], - return_types: &[ir::Type], + switch_arg_types: &[WasmValType], + return_types: &[WasmValType], ) -> WasmResult> { let vmctx = env.vmctx_val(&mut builder.cursor()); @@ -1997,6 +2219,7 @@ pub(crate) fn translate_switch<'a>( // `switcher_contref`) to the immediate child (called // `switcher_contref_last_ancestor`) of the stack with the corresponding // handler (saved in `handler_stack_chain`). + let return_values_need_gc_ref_markers = types_need_gc_ref_markers(return_types); let ( switcher_contref, switcher_contobj, @@ -2023,11 +2246,17 @@ pub(crate) fn translate_switch<'a>( // reference. let values = switcher_contref.values(env, builder); let required_capacity = u32::try_from(std::cmp::max(1, return_types.len())).unwrap(); - env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot( + let existing_storage = env.stack_switching_values_storage; + env.stack_switching_values_storage = Some(values.prepare_stack_storage( env, builder, required_capacity, - env.stack_switching_values_buffer, + // No GC ref marking is required at this point. Upon + // return to this site `vmcontref_store_payloads` will + // store the markers, if needed. + &[], + return_values_need_gc_ref_markers, + existing_storage, )); let switcher_contref_csi = switcher_contref.common_stack_information(env, builder); @@ -2073,7 +2302,13 @@ pub(crate) fn translate_switch<'a>( let (switchee_contref_csi, switchee_contref_last_ancestor) = { let mut combined_payloads = switch_args.to_vec(); combined_payloads.push(switcher_contobj); - vmcontref_store_payloads(env, builder, &combined_payloads, switchee_contref.address); + vmcontref_store_payloads( + env, + builder, + &combined_payloads, + switch_arg_types, + switchee_contref.address, + ); let switchee_contref_csi = switchee_contref.common_stack_information(env, builder); switchee_contref_csi.set_state_running(env, builder); @@ -2202,11 +2437,20 @@ pub(crate) fn translate_switch<'a>( // After switching back to the original stack, either load the values // supplied by an ordinary resume or throw the injected exception. - load_resume_values_or_throw( + let return_values = load_resume_values_or_throw( env, builder, ControlEffect::from_u64(result), switcher_contref, return_types, - ) + ); + + if return_values_need_gc_ref_markers { + let zero = builder.ins().iconst(env.pointer_type(), 0); + switcher_contref + .values(env, builder) + .set_gc_ref_data(env, builder, zero); + } + + return_values } diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index 088e0f2bd488..376d4c29cda7 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -3158,25 +3158,14 @@ pub fn translate_operator( let arg_count = src_types.len() - dst_arity; let arg_types = &src_types[0..arg_count]; - for arg_type in arg_types { - // We can't bind GC objects using cont.bind at the moment: We - // don't have the necessary infrastructure to traverse the - // buffers used by cont.bind when looking for GC roots. Thus, - // this crude check ensures that these buffers can never contain - // GC roots to begin with. - if arg_type.is_vmgcref_type_and_not_i31() { - return Err(wasmtime_environ::WasmError::Unsupported( - "cont.bind does not support GC types at the moment".into(), - )); - } - } - let (original_contobj, args) = environ.stacks.peekn(arg_count + 1).split_last().unwrap(); let original_contobj = *original_contobj; let args = args.to_vec(); - let new_contobj = environ.translate_cont_bind(builder, original_contobj, &args); + let arg_types = arg_types.to_vec(); + let new_contobj = + environ.translate_cont_bind(builder, original_contobj, &args, &arg_types); environ.stacks.popn(arg_count + 1); environ.stacks.push1(new_contobj); @@ -3184,17 +3173,18 @@ pub fn translate_operator( Operator::Suspend { tag_index } => { let tag_index = TagIndex::from_u32(*tag_index); let param_types = environ.tag_params(tag_index).to_vec(); - let return_types: SmallVec<[_; 8]> = environ - .tag_returns(tag_index) - .iter() - .map(|ty| crate::value_type(environ.isa(), *ty)) - .collect(); + let return_types = environ.tag_returns(tag_index).to_vec(); let params = environ.stacks.peekn(param_types.len()).to_vec(); let param_count = params.len(); - let return_values = - environ.translate_suspend(builder, tag_index.as_u32(), ¶ms, &return_types)?; + let return_values = environ.translate_suspend( + builder, + tag_index.as_u32(), + ¶ms, + ¶m_types, + &return_types, + )?; environ.stacks.popn(param_count); environ.stacks.pushn(&return_values); @@ -3316,45 +3306,43 @@ pub fn translate_operator( tag_index, } => { // Arguments of the continuation we are going to switch to - let continuation_argument_types: SmallVec<[_; 8]> = environ + let switch_arg_types: SmallVec<[_; 8]> = environ .continuation_arguments(TypeIndex::from_u32(*cont_type_index)) .to_smallvec(); // Arity includes the continuation argument - let arity = continuation_argument_types.len(); + let arity = switch_arg_types.len(); let (contobj, switch_args) = environ.stacks.peekn(arity).split_last().unwrap(); let contobj = *contobj; let switch_args = switch_args.to_vec(); // Type of the continuation we are going to create by suspending the // currently running stack - let current_continuation_type = continuation_argument_types.last().unwrap(); + let current_continuation_type = switch_arg_types.last().unwrap(); let current_continuation_type = current_continuation_type.unwrap_ref_type(); // Argument types of current_continuation_type. These will in turn // be the types of the arguments we receive when someone switches // back to this switch instruction - let current_continuation_arg_types: SmallVec<[_; 8]> = - match current_continuation_type.heap_type { - WasmHeapType::ConcreteCont(index) => { - let mti = index - .as_module_type_index() - .expect("expected module-local type index"); - - environ - .continuation_arguments_from_interned(mti) - .iter() - .map(|ty| crate::value_type(environ.isa(), *ty)) - .collect() - } - _ => panic!("Invalid type on switch"), - }; + let return_types: SmallVec<[_; 8]> = match current_continuation_type.heap_type { + WasmHeapType::ConcreteCont(index) => { + let mti = index + .as_module_type_index() + .expect("expected module-local type index"); + + environ + .continuation_arguments_from_interned(mti) + .to_smallvec() + } + _ => panic!("Invalid type on switch"), + }; let switch_return_values = environ.translate_switch( builder, *tag_index, contobj, &switch_args, - ¤t_continuation_arg_types, + &switch_arg_types, + &return_types, )?; environ.stacks.popn(arity); diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index 6da4e3b2cd25..a1674422ba8c 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -154,7 +154,7 @@ macro_rules! foreach_builtin_function { // Creates a new continuation from a funcref. #[cfg(feature = "stack-switching")] - cont_new(vmctx: vmctx, r: pointer, param_count: u32, result_count: u32) -> pointer; + cont_new(vmctx: vmctx, r: pointer, param_count: u32, result_count: u32, gc_refs: u32) -> pointer; // Return the instance ID for a given vmctx. #[cfg(feature = "gc")] @@ -170,6 +170,23 @@ macro_rules! foreach_builtin_function { // Process a debug breakpoint. breakpoint(vmctx: vmctx) -> bool; + + // Intern a continuation reference into the GC heap's side table. + #[cfg(all(feature = "gc", feature = "stack-switching"))] + intern_contref_for_gc_heap( + vmctx: vmctx, + contref: pointer, + revision: pointer + ) -> u64; + + // Resolve an interned continuation-reference ID and write its + // pointer and revision witness to `result`. + #[cfg(all(feature = "gc", feature = "stack-switching"))] + get_interned_contref( + vmctx: vmctx, + contref_id: u32, + result: pointer + ) -> bool; } }; } @@ -378,6 +395,7 @@ impl BuiltinFunctionIndex { (@get ref_func pointer) => (return None); (@get table_get_lazy_init_func_ref pointer) => (return None); (@get intern_func_ref_for_gc_heap u64) => (return None); + (@get intern_contref_for_gc_heap u64) => (return None); (@get is_subtype u32) => (return None); (@get ceil_f32 f32) => (return None); (@get ceil_f64 f64) => (return None); diff --git a/crates/environ/src/stack_switching.rs b/crates/environ/src/stack_switching.rs index 8f43cbdd847c..a0d1c9d7def3 100644 --- a/crates/environ/src/stack_switching.rs +++ b/crates/environ/src/stack_switching.rs @@ -33,6 +33,10 @@ pub const STACK_STATE_TRAPPED_DISCRIMINANT: u32 = 5; /// Discriminant of variant `Return` in /// `runtime::vm::ControlEffect`. pub const CONTROL_EFFECT_RETURN_DISCRIMINANT: u32 = 0; + +/// Marker for a continuation payload slot that contains a GC-managed +/// reference and must be traced while the continuation is suspended. +pub const CONTINUATION_PAYLOAD_GC_REF: u8 = 1; /// Discriminant of variant `Resume` in /// `runtime::vm::ControlEffect`. pub const CONTROL_EFFECT_RESUME_DISCRIMINANT: u32 = 1; diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index f4f6b33b14cc..6a2c71e6ca5c 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -526,6 +526,23 @@ pub trait PtrSize { 8 + self.size() } + // Offsets within `VMPayloads` + + /// Return the offset of `VMPayloads::buffer`. + fn vmpayloads_buffer(&self) -> u8 { + 0 + } + + /// Return the offset of `VMPayloads::gc_ref_data`. + fn vmpayloads_gc_ref_data(&self) -> u8 { + self.size_of_vmhostarray() + } + + /// Return the size of `VMPayloads`. + fn size_of_vmpayloads(&self) -> u8 { + self.size_of_vmhostarray() + if cfg!(feature = "gc") { self.size() } else { 0 } + } + // Offsets within `VMCommonStackInformation` /// Return the offset of `VMCommonStackInformation::limits`. @@ -622,7 +639,7 @@ pub trait PtrSize { /// Return the offset of `VMContRef::values`. fn vmcontref_values(&self) -> u8 { - self.vmcontref_args() + self.size_of_vmhostarray() + self.vmcontref_args() + self.size_of_vmpayloads() } /// Return the offset of the `over_approximated_stack_roots` field within diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rs index 364249346c2a..578e96d52231 100644 --- a/crates/wasmtime/src/runtime/store/gc.rs +++ b/crates/wasmtime/src/runtime/store/gc.rs @@ -795,22 +795,37 @@ impl StoreOpaque { #[cfg(feature = "stack-switching")] fn trace_wasm_continuation_roots(&mut self, gc_roots_list: &mut GcRootsList) { - use crate::vm::VMStackState; + use crate::vm::{VMPayloads, VMStackState}; + + unsafe fn trace_payload_roots(gc_roots_list: &mut GcRootsList, payloads: &VMPayloads) { + let gc_ref_data = payloads.gc_ref_data; + let payloads = &payloads.buffer; + assert!(payloads.length <= payloads.capacity); + if gc_ref_data.is_null() { + return; + } + + for index in 0..usize::try_from(payloads.length).unwrap() { + let marker = unsafe { gc_ref_data.add(index).read() }; + if marker == wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF { + let slot = unsafe { payloads.data.add(index).cast::() }; + unsafe { + StoreOpaque::trace_wasm_stack_slot(gc_roots_list, slot); + } + } + } + } log::trace!("Begin trace GC roots :: continuations"); for continuation in &self.continuations { let state = continuation.common_stack_information.state; - // FIXME(frank-emrich) In general, it is not enough to just trace - // through the stacks of continuations; we also need to look through - // their `cont.bind` arguments. However, we don't currently have - // enough RTTI information to check if any of the values in the - // buffers used by `cont.bind` are GC values. As a workaround, note - // that we currently disallow cont.bind-ing GC values altogether. - // This way, it is okay not to check them here. match state { VMStackState::Suspended => { + unsafe { + trace_payload_roots(gc_roots_list, &continuation.values); + } Backtrace::trace_suspended_continuation(self, continuation.deref(), |frame| { Self::trace_wasm_stack_frame(self.modules(), gc_roots_list, frame); core::ops::ControlFlow::Continue(()) @@ -824,8 +839,11 @@ impl StoreOpaque { // either case things should be handled correctly when traversing // further along in the chain, nothing required at this point. } - VMStackState::Fresh | VMStackState::Returned | VMStackState::Trapped => { - // Fresh/terminal continuations have no live GC values on their stack. + VMStackState::Fresh => unsafe { + trace_payload_roots(gc_roots_list, &continuation.args); + }, + VMStackState::Returned | VMStackState::Trapped => { + // Terminal continuations have no live GC values. } } } diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rs index 5b0aae724863..28bf087928fa 100644 --- a/crates/wasmtime/src/runtime/vm/gc.rs +++ b/crates/wasmtime/src/runtime/vm/gc.rs @@ -8,6 +8,8 @@ mod disabled; #[cfg(not(feature = "gc"))] pub use disabled::*; +#[cfg(feature = "stack-switching")] +mod cont_ref; mod data; mod func_ref; mod gc_ref; @@ -15,6 +17,8 @@ mod gc_runtime; mod host_data; mod i31; +#[cfg(feature = "stack-switching")] +pub use cont_ref::*; pub use data::*; pub use func_ref::*; pub use gc_ref::*; @@ -54,6 +58,10 @@ pub struct GcStore { /// The function-references table for this GC heap. pub func_ref_table: FuncRefTable, + /// The continuation references table for this GC heap. + #[cfg(feature = "stack-switching")] + pub cont_ref_table: ContRefTable, + /// The total allocated bytes recorded after the last GC collection. /// `None` if no collection has been performed yet. Used by the /// grow-or-collect heuristic. @@ -96,6 +104,8 @@ impl GcStore { ) -> Self { let host_data_table = ExternRefHostDataTable::default(); let func_ref_table = FuncRefTable::default(); + #[cfg(feature = "stack-switching")] + let cont_ref_table = ContRefTable::default(); let _ = &gc_zeal_alloc_counter; @@ -104,6 +114,8 @@ impl GcStore { gc_heap, host_data_table, func_ref_table, + #[cfg(feature = "stack-switching")] + cont_ref_table, last_post_gc_allocated_bytes: None, #[cfg(gc_zeal)] gc_zeal_alloc_counter, diff --git a/crates/wasmtime/src/runtime/vm/gc/cont_ref.rs b/crates/wasmtime/src/runtime/vm/gc/cont_ref.rs new file mode 100644 index 000000000000..ece4b6b03030 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/gc/cont_ref.rs @@ -0,0 +1,58 @@ +//! Side table for continuation references stored in the GC heap. +//! +//! Continuation references are sixteen bytes values, while every +//! reference field in the GC heap is four bytes. GC objects therefore +//! store an ID into this table rather than storing a `VMContObj` +//! directly. + +use crate::{Result, bail_bug, hash_map::HashMap, vm::VMContObj}; +use wasmtime_core::{ + alloc::PanicOnOom, + slab::{Id, Slab}, +}; + +/// Side table mapping the IDs stored in GC fields to continuation +/// objects. +/// +/// Raw ID zero is reserved for a null continuation reference. +/// Non-null IDs are one greater than their underlying slab IDs. +#[derive(Default)] +pub struct ContRefTable { + interned: HashMap, + slab: Slab, +} + +impl ContRefTable { + /// Intern a continuation object and return the ID to store in the + /// GC heap. Null continuation references use the reserved ID + /// zero. + /// + /// # Safety + /// + /// A non-null continuation's `VMContRef` pointer must remain valid for the + /// duration of this table's lifetime. + pub unsafe fn intern(&mut self, contobj: Option) -> u32 { + let Some(contobj) = contobj else { + return 0; + }; + + *self.interned.entry(contobj).or_insert_with(|| { + // TODO(dhil): Handle allocation failure here rather than panicking. + let id = self.slab.alloc(contobj).panic_on_oom().into_raw(); + id.checked_add(1).unwrap() + }) + } + + /// Resolve an ID loaded from the GC heap. + pub fn get(&self, raw: u32) -> Result> { + if raw == 0 { + return Ok(None); + } + + let id = Id::from_raw(raw - 1); + match self.slab.get(id).copied() { + Some(contobj) => Ok(Some(contobj)), + None => bail_bug!("bad continuation-reference table ID"), + } + } +} diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index 6964e405d62e..9e67e78396de 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -604,6 +604,59 @@ fn get_interned_func_ref( Ok(func_ref.map_or(core::ptr::null_mut(), |f| f.as_ptr().cast())) } +// Intern a continuation reference into the GC heap's side table. +// +// This libcall may not GC. +#[cfg(all(feature = "gc", feature = "stack-switching"))] +unsafe fn intern_contref_for_gc_heap( + store: &mut dyn VMStore, + _instance: InstanceId, + contref: *mut u8, + revision: *mut u8, +) -> Result { + use crate::store::AutoAssertNoGc; + + let mut store = AutoAssertNoGc::new(store.store_opaque_mut()); + let contobj = unsafe { crate::vm::VMContObj::from_raw_parts(contref, revision.addr()) }; + let id = unsafe { store.require_gc_store_mut()?.cont_ref_table.intern(contobj) }; + Ok(id) +} + +// Resolve a continuation reference ID loaded from the GC heap and +// write its 16 bytes value into caller provided storage `out_result`. +// +// This libcall may not GC. +#[cfg(all(feature = "gc", feature = "stack-switching"))] +unsafe fn get_interned_contref( + store: &mut dyn VMStore, + _instance: InstanceId, + contref_id: u32, + out_result: *mut u8, +) -> Result<()> { + use crate::store::AutoAssertNoGc; + + #[repr(C)] + struct RawContObj { + contref: *mut u8, + revision: usize, + } + + let store = AutoAssertNoGc::new(store.store_opaque_mut()); + let contobj = store.unwrap_gc_store().cont_ref_table.get(contref_id)?; + let raw = match contobj { + Some(contobj) => RawContObj { + contref: contobj.contref.as_ptr().cast(), + revision: contobj.revision, + }, + None => RawContObj { + contref: core::ptr::null_mut(), + revision: 0, + }, + }; + unsafe { out_result.cast::().write(raw) }; + Ok(()) +} + #[cfg(feature = "gc")] fn is_subtype( store: &mut dyn VMStore, @@ -1119,9 +1172,25 @@ fn cont_new( func: *mut u8, param_count: u32, result_count: u32, + gc_refs: u32, ) -> Result> { - let ans = - crate::vm::stack_switching::cont_new(store, instance, func, param_count, result_count)?; + let ans = if gc_refs != 0 { + crate::vm::stack_switching::cont_new::( + store, + instance, + func, + param_count, + result_count, + )? + } else { + crate::vm::stack_switching::cont_new::( + store, + instance, + func, + param_count, + result_count, + )? + }; Ok(Some(AllocationSize(ans.cast::() as usize))) } diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index 29607771dd92..1918399f7ed1 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -50,7 +50,7 @@ pub const CONTROL_EFFECT_TRAP_ENCODING: u64 = /// (i.e., the one pointed to by the VMContObj) has a pointer to the /// other end of the chain (i.e., its last ancestor). #[repr(C)] -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct VMContObj { pub contref: NonNull, pub revision: usize, @@ -164,6 +164,14 @@ impl VMStackLimits { #[derive(Debug, Clone)] /// Reference to a stack-allocated buffer ("array"), storing data of some type /// `T`. +/// +/// The core invariants of this type are: +/// +/// * `length <= capacity`. +/// * When `capacity > 0`, `data` points to storage for at least `capacity` +/// properly aligned values of `T` and remains valid while this descriptor is +/// in use. +/// * Only the first `length` entries are initialized and logically occupied. pub struct VMHostArray { /// Number of currently occupied slots. pub length: u32, @@ -190,10 +198,55 @@ impl VMHostArray { } } -/// Type used for passing payloads to and from continuations. The actual type -/// argument should be wasmtime::runtime::vm::vmcontext::ValRaw, but we don't -/// have access to that here. -pub type VMPayloads = VMHostArray; +/// Payload values exchanged with a continuation and the metadata needed to +/// trace GC references among them. +/// +/// In addition to the [`VMHostArray`] invariants, this type maintains the +/// following core invariants: +/// +/// * A null `gc_ref_data` means that no occupied slot contains a +/// GC-managed reference that needs tracing. +/// * A non-null `gc_ref_data` points to at least `buffer.capacity` +/// marker bytes and remains valid for as long as `buffer.data` is +/// valid. +/// * For every occupied slot, its marker records whether the slot +/// contains a GC-managed reference. Unoccupied marker bytes are +/// ignored. +/// * A non-null `gc_ref_data` is invalidated when the storage is +/// cleared or the containing continuation enters a terminal state in +/// which payloads are no longer traced. +/// +/// The actual buffer element type should be +/// `wasmtime::runtime::vm::vmcontext::ValRaw`, but we don't have +/// access to that type here. A `u128` supplies the required 16-byte +/// slot size, and the type parameter does not affect the layout of +/// the `VMHostArray` descriptor. +#[repr(C)] +#[derive(Debug, Clone)] +pub struct VMPayloads { + pub buffer: VMHostArray, + + #[cfg(feature = "gc")] + pub gc_ref_data: *mut u8, +} + +impl VMPayloads { + pub fn empty() -> Self { + Self { + buffer: VMHostArray::empty(), + #[cfg(feature = "gc")] + gc_ref_data: core::ptr::null_mut(), + } + } + + pub fn clear(&mut self) { + self.buffer.clear(); + #[cfg(feature = "gc")] + { + self.gc_ref_data = core::ptr::null_mut(); + } + } +} /// Type for a list of handlers, represented by the handled tag. Thus, the /// stored data is actually `*mut VMTagDefinition`, but we don't have access to @@ -227,7 +280,7 @@ pub struct VMContRef { /// 1. The arguments to the function passed to cont.new /// 2. The return values of that function /// - /// Note that the actual data buffer (i.e., the one `args.data` points + /// Note that the actual data buffer (i.e., the one `args.buffer.data` points /// to) is always allocated on this continuation's stack. pub args: VMPayloads, @@ -238,7 +291,7 @@ pub struct VMContRef { /// - Pass payloads to a continuation using cont.bind or resume /// - Pass payloads to the continuation being switched to when using switch. /// - /// Note that the actual data buffer (i.e., the one `values.data` points + /// Note that the actual data buffer (i.e., the one `values.buffer.data` points /// to) is always allocated on this continuation's stack. pub values: VMPayloads, @@ -310,7 +363,7 @@ unsafe impl Sync for VMContRef {} /// Implements `cont.new` instructions (i.e., creation of continuations). #[cfg(feature = "stack-switching")] #[inline(always)] -pub fn cont_new( +pub fn cont_new( store: &mut dyn crate::vm::VMStore, instance: crate::store::InstanceId, func: *mut u8, @@ -333,9 +386,8 @@ pub fn cont_new( // The initialization function will allocate the actual args/return value buffer and // update this object (if needed). - let contref_args_ptr = &mut contref.args as *mut _ as *mut VMHostArray; - - contref.stack.initialize( + let contref_args_ptr = &mut contref.args as *mut VMPayloads; + contref.stack.initialize::( func.cast::(), caller_vmctx.as_ptr(), contref_args_ptr, @@ -648,6 +700,25 @@ mod tests { ); } + #[test] + fn check_vm_payloads_offsets() { + let module = Module::new(StaticModuleIndex::from_u32(0)); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + size_of::(), + usize::from(offsets.ptr.size_of_vmpayloads()) + ); + assert_eq!( + offset_of!(VMPayloads, buffer), + usize::from(offsets.ptr.vmpayloads_buffer()) + ); + #[cfg(feature = "gc")] + assert_eq!( + offset_of!(VMPayloads, gc_ref_data), + usize::from(offsets.ptr.vmpayloads_gc_ref_data()) + ); + } + #[test] fn check_vm_contobj_offsets() { let module = Module::new(StaticModuleIndex::from_u32(0)); diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs index 14984b30ed43..a0c5857ffce8 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs @@ -4,8 +4,8 @@ use crate::Result; use core::ops::Range; -use crate::runtime::vm::stack_switching::VMHostArray; -use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; +use crate::runtime::vm::stack_switching::VMPayloads; +use crate::runtime::vm::{VMContext, VMFuncRef}; cfg_select! { all(feature = "stack-switching", unix, target_arch = "x86_64") => { @@ -95,19 +95,21 @@ impl VMContinuationStack { /// Initializes this stack, such that it will execute the function denoted /// by `func_ref`. `parameter_count` and `return_value_count` must be the /// corresponding number of parameters and return values of `func_ref`. - /// `args` must point to the `args` field of the `VMContRef` owning this pointer. + /// `args` must point to the `args` field of the `VMContRef` owning this + /// stack. When `GC_REFS` is true, its `gc_ref_data` field receives the + /// corresponding root-marker buffer. /// /// It will be updated by this function to correctly describe /// the buffer used by this function for its arguments and return values. - pub fn initialize( + pub fn initialize( &self, func_ref: *const VMFuncRef, caller_vmctx: *mut VMContext, - args: *mut VMHostArray, + args: *mut VMPayloads, parameter_count: u32, return_value_count: u32, ) -> Result<()> { - self.0.initialize( + self.0.initialize::( func_ref, caller_vmctx, args, diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs index fdff6186987f..7e6ba89936e0 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs @@ -1,8 +1,8 @@ use crate::Result; use core::ops::Range; -use crate::runtime::vm::stack_switching::VMHostArray; -use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; +use crate::runtime::vm::stack_switching::VMPayloads; +use crate::runtime::vm::{VMContext, VMFuncRef}; /// Making sure that this has the same size as the non-dummy version, to /// make some tests happy. @@ -55,11 +55,11 @@ impl VMContinuationStack { panic!("Stack switching disabled or not implemented on this platform") } - pub fn initialize( + pub fn initialize( &self, _func_ref: *const VMFuncRef, _caller_vmctx: *mut VMContext, - _args: *mut VMHostArray, + _args: *mut VMPayloads, _parameter_count: u32, _return_value_count: u32, ) -> Result<()> { diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs index ee3152a49984..779db376936d 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs @@ -13,7 +13,18 @@ //! 0xAfe0 +-----------------------+ //! | args buffer, size: | //! | (16 * args_capacity) | -//! 0xAfc0 +-----------------------+ <- below: beginning of usable stack space +//! 0xAfc0 +-----------------------+ +//! | optional GC-reference | +//! | markers (16B-aligned) | +//! 0xAfb0 +-----------------------+ <- RSP after consuming the launch record +//! | func_ref | +//! 0xAfa8 +-----------------------+ +//! | caller_vmctx | +//! 0xAfa0 +-----------------------+ +//! | args descriptor ptr | +//! 0xAf98 +-----------------------+ +//! | return value count | +//! 0xAf90 +-----------------------+ <- initial RSP; consumable launch record //! | | (16-byte aligned) //! | | //! ~ ... ~ <- actual native stack space to use @@ -57,6 +68,15 @@ //! the args buffer is args_capacity * 16 bytes. The start address (0xAfc0 in //! the example above, thus assuming args_capacity = 2) is saved as the `data` //! field of the VMContRef's `args` object. +//! +//! The example above assumes `args_capacity = 2` and that GC-reference marker +//! storage is present. The argument and marker buffers are persistent launch +//! storage: they remain above RSP while the continuation executes. The four +//! words below them form a consumable launch record. The +//! `wasmtime_continuation_start` trampoline begins with RSP at 0xAf90 and pops +//! those words; afterwards RSP is 0xAfb0, immediately below the persistent +//! buffers. Calls and stack frames then grow towards lower addresses, reusing +//! the consumed launch record without overwriting the buffers. use core::ptr::NonNull; use std::io; @@ -64,7 +84,7 @@ use std::ops::Range; use std::ptr; use crate::prelude::*; -use crate::runtime::vm::stack_switching::VMHostArray; +use crate::runtime::vm::stack_switching::{VMHostArray, VMPayloads}; use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; #[derive(Debug, PartialEq, Eq)] @@ -191,14 +211,17 @@ impl VMContinuationStack { /// /// Concretely, switching to the stack prepared by this function /// causes that we enter `wasmtime_continuation_start`, which then in turn - /// calls `fiber_start` with the following arguments: - /// TOS, func_ref, caller_vmctx, args_ptr, args_capacity + /// calls `fiber_start` with the following arguments: + /// func_ref, caller_vmctx, args_ptr, return_value_count /// - /// Note that at this point we also allocate the args buffer - /// (see picture at the top of this file). + /// Note that at this point we also allocate persistent launch storage for + /// the args buffer and, when `GC_REFS` is true, its parallel + /// GC-reference-marker buffer (see picture at the top of this file). /// We define `args_capacity` as the max of parameter and return value count. - /// Then the size s of the actual buffer size is calculated as follows: - /// s = size_of(ValRaw) * `args_capacity`, + /// Their combined, 16-byte-aligned size `s` is calculated as follows when + /// `GC_REFS` is true: + /// s = size_of(ValRaw) * `args_capacity` + /// + align_up(`args_capacity`, 16), /// /// Note that this value is used below, and we may have s = 0. /// @@ -215,20 +238,27 @@ impl VMContinuationStack { /// -0x20 | args_capacity /// /// - /// The data stored behind the args buffer is as follows: + /// The consumable launch record stored immediately below the persistent + /// buffer(s) is as follows: /// /// Offset from | /// TOS | Contents /// ---------------|------------------------------------------------------- /// -0x28 - s | func_ref /// -0x30 - s | caller_vmctx - /// -0x38 - s | args (of type *mut ArrayRef) + /// -0x38 - s | args (of type *mut VMHostArray) /// -0x40 - s | return_value_count - pub fn initialize( + /// + /// The saved RSP points at `TOS - 0x40 - s`. On entry, + /// `wasmtime_continuation_start` pops the four launch-record words, leaving + /// RSP at `TOS - 0x20 - s`, immediately below the persistent buffers. + /// Subsequent stack frames grow towards lower addresses and may reuse the + /// consumed launch-record storage. + pub fn initialize( &self, func_ref: *const VMFuncRef, caller_vmctx: *mut VMContext, - args: *mut VMHostArray, + args: *mut VMPayloads, parameter_count: u32, return_value_count: u32, ) -> Result<()> { @@ -240,23 +270,51 @@ impl VMContinuationStack { target.write(value) }; - let args_ref = &mut *args; + let payloads = &mut *args; + let args_ref = &mut payloads.buffer; let args_capacity = std::cmp::max(parameter_count, return_value_count); // The args object must currently be empty. debug_assert_eq!(args_ref.capacity, 0); debug_assert_eq!(args_ref.length, 0); - let total_control_size = usize::try_from(args_capacity)? + let args_data_size = usize::try_from(args_capacity)? .checked_mul(std::mem::size_of::()) - .and_then(|s| s.checked_add(0x40)) .ok_or_else(|| { format_err!( "continuation function type with {args_capacity} args \ overflows stack control data size calculation" ) })?; - let args_data_size = total_control_size - 0x40; - + // Keep the fixed startup data 16-byte aligned. + let gc_refs_data_size = if cfg!(feature = "gc") && GC_REFS { + usize::try_from(args_capacity)? + .checked_add(15) + .map(|s| s & !15) + .ok_or_else(|| { + format_err!( + "continuation function type with {args_capacity} args \ + overflows stack control data size calculation" + ) + })? + } else { + 0 + }; + let dynamic_data_size = args_data_size + .checked_add(gc_refs_data_size) + .ok_or_else(|| { + format_err!( + "continuation function type with {args_capacity} args \ + overflows stack control data size calculation" + ) + })?; + let total_control_size = dynamic_data_size + .checked_add(0x40) + .ok_or_else(|| { + format_err!( + "continuation function type with {args_capacity} args \ + overflows stack control data size calculation" + ) + })?; // Ensure the control data (fixed header + args buffer) fits // within the usable stack space. For Mmap allocations, // self.len includes the guard page, which is not writable. @@ -279,9 +337,20 @@ impl VMContinuationStack { } else { tos.sub(0x20 + args_data_size) }; - args_ref.capacity = args_capacity; - args_ref.data = args_data_ptr.cast::(); + args_ref.data = args_data_ptr.cast::(); + #[cfg(feature = "gc")] + if GC_REFS { + let data = if args_capacity == 0 { + ptr::null_mut() + } else { + tos.sub(0x20 + dynamic_data_size) + }; + if args_capacity > 0 { + data.write_bytes(0, usize::try_from(args_capacity)?); + } + payloads.gc_ref_data = data; + } let to_store = [ // Data near top of stack: @@ -290,10 +359,16 @@ impl VMContinuationStack { (0x18, tos.sub(total_control_size).addr()), (0x20, usize::try_from(args_capacity)?), // Data after the args buffer: - (0x28 + args_data_size, func_ref.addr()), - (0x30 + args_data_size, caller_vmctx.addr()), - (0x38 + args_data_size, args.addr()), - (0x40 + args_data_size, usize::try_from(return_value_count)?), + (0x28 + dynamic_data_size, func_ref.addr()), + (0x30 + dynamic_data_size, caller_vmctx.addr()), + ( + 0x38 + dynamic_data_size, + (args_ref as *mut VMHostArray).addr(), + ), + ( + 0x40 + dynamic_data_size, + usize::try_from(return_value_count)?, + ), ]; for (offset, data) in to_store { @@ -324,7 +399,7 @@ impl Drop for VMContinuationStack { unsafe extern "C" fn fiber_start( func_ref: *mut VMFuncRef, caller_vmctx: *mut VMContext, - args: *mut VMHostArray, + args: *mut VMHostArray, return_value_count: u32, ) -> bool { unsafe { @@ -334,8 +409,11 @@ unsafe extern "C" fn fiber_start( let params_and_returns: NonNull<[ValRaw]> = if args.capacity == 0 { NonNull::from(&[]) } else { - std::slice::from_raw_parts_mut(args.data, usize::try_from(args.capacity).unwrap()) - .into() + std::slice::from_raw_parts_mut( + args.data.cast::(), + usize::try_from(args.capacity).unwrap(), + ) + .into() }; // NOTE(frank-emrich) The usage of the `caller_vmctx` is probably not diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs index bc4f60a7d6a9..69337194d2c1 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs @@ -34,6 +34,7 @@ pub fn wasmtime_continuation_start_address() -> *const () { // values in various registers when execution of wasmtime_continuation_start begins: // // RSP: TOS - 0x40 - (16 * `args_capacity`) +// - align_up(`args_capacity`, 16) // RBP: TOS - 0x10 #[unsafe(naked)] diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index 0b40ebadf05e..eb3d8962e1fa 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -335,10 +335,15 @@ impl Backtrace { let mut continuations_iter = unsafe { chain.into_continuation_iter() }.peekable(); while let Some(continuation_ptr) = continuations_iter.next() { - let parent_limits_ptr = stack_limits_iter - .next() - .expect("expected one more VMStackLimits than continuations"); let continuation = unsafe { &*continuation_ptr }; + let Some(parent_limits_ptr) = stack_limits_iter.next() else { + // A detached suspended continuation chain ends in `Absent`, + // unlike a running chain which ends in an initial stack. The + // currently suspended stack was already traced above, and the + // last continuation has no parent stack left to trace. + debug_assert_eq!(continuation.parent_chain, VMStackChain::Absent); + break; + }; let parent_limits = unsafe { &*parent_limits_ptr }; // The parent of `continuation` if present and not the last in the chain. diff --git a/tests/all/stack_switching.rs b/tests/all/stack_switching.rs index dfc33714f8f1..3a075d471924 100644 --- a/tests/all/stack_switching.rs +++ b/tests/all/stack_switching.rs @@ -339,6 +339,71 @@ fn inter_instance_suspend() -> Result<()> { Ok(()) } +/// Regression test for https://github.com/bytecodealliance/wasmtime/issues/13750. +/// +/// A GC reference returned through a continuation payload must retain +/// its stack-map metadata while it is live on the operand stack +/// across subsequent GC safepoints. +#[cfg_attr(any(asan, miri), ignore)] +#[test] +fn continuation_result_gc_ref_on_operand_stack() -> Result<()> { + let mut config = Config::new(); + config + .wasm_stack_switching(true) + .wasm_exceptions(true) + .wasm_function_references(true) + .wasm_gc(true) + .collector(Collector::Copying); + + #[cfg(gc_zeal)] + { + let _ = config.gc_zeal_alloc_counter(std::num::NonZeroU32::new(1)); + } + + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (type $S (struct (field $x i32))) + (type $A (array i64)) + (type $ft (func (result (ref $S)))) + (type $ct (cont $ft)) + + (func $target (result (ref $S)) + (struct.new $S (i32.const 0x12345678))) + (elem declare func $target) + + (func (export "run") (result i32) + (local $k (ref null $ct)) + (local $i i32) + (local.set $k (cont.new $ct (ref.func $target))) + + ;; Keep the continuation's GC-reference result on the operand + ;; stack, rather than storing it in a GC-typed local. + (resume $ct (local.get $k)) + + ;; Force collections while that operand-stack value is the + ;; only live reference to the struct. + (local.set $i (i32.const 0)) + (loop $again + (drop (array.new_default $A (i32.const 256))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br_if $again + (i32.lt_u (local.get $i) (i32.const 2000)))) + + (struct.get $S $x)) + ) + "#, + )?; + + let mut store = Store::new(&engine, ()); + let instance = Instance::new(&mut store, &module, &[])?; + let run = instance.get_typed_func::<(), i32>(&mut store, "run")?; + assert_eq!(run.call(&mut store, ())?, 0x12345678); + Ok(()) +} + /// Tests interaction with host functions. Note that the interaction with host /// functions and traps is covered by the module `traps` further down. mod host { diff --git a/tests/disas/stack-switching/resume-suspend-data-passing.wat b/tests/disas/stack-switching/resume-suspend-data-passing.wat index 4ca71c42341c..9e30ecd38920 100644 --- a/tests/disas/stack-switching/resume-suspend-data-passing.wat +++ b/tests/disas/stack-switching/resume-suspend-data-passing.wat @@ -54,7 +54,7 @@ ;; block0(v0: i64, v1: i64): ;; @003c v3 = iconst.i32 10 ;; @0044 v7 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @0044 v31 = iconst.i64 136 +;; @0044 v31 = iconst.i64 144 ;; @0044 v34 = stack_addr.i64 ss0 ;; @0044 v40 = iconst.i64 32 ;; @0044 v37 = iconst.i64 0 @@ -111,8 +111,8 @@ ;; block8: ;; @0044 store.i64 notrap aligned region3 v11, v9+80 ;; v93 = iconst.i32 1 -;; v94 = iconst.i64 136 -;; v95 = iadd.i64 v9, v94 ; v94 = 136 +;; v94 = iconst.i64 144 +;; v95 = iadd.i64 v9, v94 ; v94 = 144 ;; @0044 store notrap aligned region3 v93, v95+4 ; v93 = 1 ;; @0044 store.i64 notrap aligned region3 v34, v95+8 ;; @0044 store.i32 notrap aligned region4 v4, v34 @@ -186,7 +186,7 @@ ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 ;; sig0 = (i64 vmctx, i32) -> i64 tail -;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail ;; sig2 = (i64 vmctx) tail ;; fn0 = colocated u805306368:6 sig0 ;; fn1 = colocated u805306368:42 sig1 @@ -197,203 +197,203 @@ ;; @0056 v2 = iconst.i32 0 ;; @0056 v3 = call fn0(v0, v2) ; v2 = 0 ;; @0058 trapz v3, user16 -;; @0058 v6 = call fn1(v0, v3, v2, v2) ; v2 = 0, v2 = 0 -;; @0058 v7 = load.i64 notrap aligned region2 v6+88 +;; @0058 v7 = call fn1(v0, v3, v2, v2, v2) ; v2 = 0, v2 = 0, v2 = 0 +;; @0058 v8 = load.i64 notrap aligned region2 v7+88 +;; @0058 v10 = uextend.i128 v8 +;; @0058 v11 = iconst.i64 64 +;; v149 = ishl v10, v11 ; v11 = 64 ;; @0058 v9 = uextend.i128 v7 -;; @0058 v10 = iconst.i64 64 -;; v148 = ishl v9, v10 ; v10 = 64 -;; @0058 v8 = uextend.i128 v6 -;; @0058 v13 = bor v148, v8 -;; @0062 v22 = iconst.i64 1 -;; @0062 v25 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @0062 v28 = iconst.i64 0 -;; @0062 v29 = iconst.i64 2 -;; @0062 v33 = iconst.i32 1 -;; @0062 v34 = iconst.i64 32 -;; @0062 v36 = iconst.i32 2 -;; @0062 v52 = iconst.i64 40 -;; @0062 v55 = stack_addr.i64 ss0 -;; @0062 v56 = iconst.i64 48 -;; @0062 v57 = iadd v0, v56 ; v56 = 48 -;; @0062 v64 = iconst.i64 96 -;; @0062 v67 = iconst.i64 -24 -;; v152 = iconst.i64 0x0001_0000_0000 -;; @005c jump block2(v13) +;; @0058 v14 = bor v149, v9 +;; @0062 v23 = iconst.i64 1 +;; @0062 v26 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @0062 v29 = iconst.i64 0 +;; @0062 v30 = iconst.i64 2 +;; @0062 v34 = iconst.i32 1 +;; @0062 v35 = iconst.i64 32 +;; @0062 v37 = iconst.i32 2 +;; @0062 v53 = iconst.i64 40 +;; @0062 v56 = stack_addr.i64 ss0 +;; @0062 v57 = iconst.i64 48 +;; @0062 v58 = iadd v0, v57 ; v57 = 48 +;; @0062 v65 = iconst.i64 96 +;; @0062 v68 = iconst.i64 -24 +;; v153 = iconst.i64 0x0001_0000_0000 +;; @005c jump block2(v14) ;; -;; block2(v14: i128): +;; block2(v15: i128): ;; @0062 jump block5 ;; ;; block5: -;; @0062 v15 = ireduce.i64 v14 -;; @0062 trapz v15, user16 -;; @0062 v20 = load.i64 notrap aligned region2 v15+88 -;; v155 = iconst.i64 64 -;; v156 = ushr.i128 v14, v155 ; v155 = 64 -;; @0062 v19 = ireduce.i64 v156 -;; @0062 v21 = icmp eq v20, v19 -;; @0062 trapz v21, user23 -;; v157 = iconst.i64 1 -;; v158 = iadd v20, v157 ; v157 = 1 -;; @0062 store notrap aligned region2 v158, v15+88 -;; @0062 v24 = load.i64 notrap aligned region2 v15+80 -;; @0062 v26 = load.i64 notrap aligned region3 v25+88 -;; @0062 v27 = load.i64 notrap aligned region3 v25+96 -;; @0062 store notrap aligned region2 v26, v24+64 -;; @0062 store notrap aligned region2 v27, v24+72 -;; v159 = iconst.i64 0 -;; @0062 store notrap aligned region2 v159, v15+80 ; v159 = 0 -;; v160 = iconst.i64 2 -;; @0062 store notrap aligned region3 v160, v25+88 ; v160 = 2 -;; @0062 store notrap aligned region3 v15, v25+96 -;; v161 = iconst.i32 1 -;; v162 = iconst.i64 32 -;; v163 = iadd v15, v162 ; v162 = 32 -;; @0062 store notrap aligned region2 v161, v163 ; v161 = 1 -;; v164 = iconst.i32 2 -;; v165 = iadd v27, v162 ; v162 = 32 -;; @0062 store notrap aligned region2 v164, v165 ; v164 = 2 -;; @0062 v42 = load.i64 notrap aligned region4 v25+72 -;; @0062 v43 = load.i64 notrap aligned region5 v25+64 -;; @0062 v44 = load.i64 notrap aligned region6 v25+80 -;; @0062 store notrap aligned region2 v42, v27+8 -;; @0062 store notrap aligned region2 v43, v27+16 -;; @0062 store notrap aligned region2 v44, v27+24 -;; @0062 v45 = load.i64 notrap aligned region1 v25+24 -;; @0062 store notrap aligned region2 v45, v27 -;; @0062 v48 = load.i64 notrap aligned region2 v15 -;; @0062 store notrap aligned region1 v48, v25+24 -;; @0062 v49 = load.i64 notrap aligned region2 v15+8 -;; @0062 store notrap aligned region4 v49, v25+72 -;; @0062 v50 = load.i64 notrap aligned region2 v15+16 -;; @0062 store notrap aligned region5 v50, v25+64 -;; @0062 v51 = load.i64 notrap aligned region2 v15+24 -;; @0062 store notrap aligned region6 v51, v25+80 -;; v166 = iconst.i64 40 -;; v167 = iadd v27, v166 ; v166 = 40 -;; @0062 store notrap aligned region2 v161, v167+4 ; v161 = 1 -;; @0062 store.i64 notrap aligned region2 v55, v167+8 -;; v168 = iadd.i64 v0, v56 ; v56 = 48 -;; @0062 store notrap aligned region7 v168, v55 -;; @0062 store notrap aligned region2 v161, v167 ; v161 = 1 -;; @0062 store notrap aligned region2 v161, v27+56 ; v161 = 1 -;; v169 = iconst.i64 96 -;; v170 = iadd v24, v169 ; v169 = 96 -;; @0062 v66 = load.i64 notrap aligned region2 v170 -;; v171 = iconst.i64 -24 -;; v172 = iadd v66, v171 ; v171 = -24 -;; v173 = iconst.i64 0x0001_0000_0000 -;; @0062 v69 = stack_switch v172, v172, v173 ; v173 = 0x0001_0000_0000 -;; @0062 v71 = load.i64 notrap aligned region3 v25+88 -;; @0062 v72 = load.i64 notrap aligned region3 v25+96 -;; @0062 store notrap aligned region3 v26, v25+88 -;; @0062 store notrap aligned region3 v27, v25+96 -;; @0062 store notrap aligned region2 v161, v165 ; v161 = 1 -;; v174 = iconst.i32 0 -;; @0062 store notrap aligned region2 v174, v167 ; v174 = 0 -;; @0062 store notrap aligned region2 v174, v167+4 ; v174 = 0 -;; @0062 store notrap aligned region2 v159, v167+8 ; v159 = 0 -;; @0062 store notrap aligned region2 v159, v27+56 ; v159 = 0 -;; @0062 brif v69, block9, block6 +;; @0062 v16 = ireduce.i64 v15 +;; @0062 trapz v16, user16 +;; @0062 v21 = load.i64 notrap aligned region2 v16+88 +;; v156 = iconst.i64 64 +;; v157 = ushr.i128 v15, v156 ; v156 = 64 +;; @0062 v20 = ireduce.i64 v157 +;; @0062 v22 = icmp eq v21, v20 +;; @0062 trapz v22, user23 +;; v158 = iconst.i64 1 +;; v159 = iadd v21, v158 ; v158 = 1 +;; @0062 store notrap aligned region2 v159, v16+88 +;; @0062 v25 = load.i64 notrap aligned region2 v16+80 +;; @0062 v27 = load.i64 notrap aligned region3 v26+88 +;; @0062 v28 = load.i64 notrap aligned region3 v26+96 +;; @0062 store notrap aligned region2 v27, v25+64 +;; @0062 store notrap aligned region2 v28, v25+72 +;; v160 = iconst.i64 0 +;; @0062 store notrap aligned region2 v160, v16+80 ; v160 = 0 +;; v161 = iconst.i64 2 +;; @0062 store notrap aligned region3 v161, v26+88 ; v161 = 2 +;; @0062 store notrap aligned region3 v16, v26+96 +;; v162 = iconst.i32 1 +;; v163 = iconst.i64 32 +;; v164 = iadd v16, v163 ; v163 = 32 +;; @0062 store notrap aligned region2 v162, v164 ; v162 = 1 +;; v165 = iconst.i32 2 +;; v166 = iadd v28, v163 ; v163 = 32 +;; @0062 store notrap aligned region2 v165, v166 ; v165 = 2 +;; @0062 v43 = load.i64 notrap aligned region4 v26+72 +;; @0062 v44 = load.i64 notrap aligned region5 v26+64 +;; @0062 v45 = load.i64 notrap aligned region6 v26+80 +;; @0062 store notrap aligned region2 v43, v28+8 +;; @0062 store notrap aligned region2 v44, v28+16 +;; @0062 store notrap aligned region2 v45, v28+24 +;; @0062 v46 = load.i64 notrap aligned region1 v26+24 +;; @0062 store notrap aligned region2 v46, v28 +;; @0062 v49 = load.i64 notrap aligned region2 v16 +;; @0062 store notrap aligned region1 v49, v26+24 +;; @0062 v50 = load.i64 notrap aligned region2 v16+8 +;; @0062 store notrap aligned region4 v50, v26+72 +;; @0062 v51 = load.i64 notrap aligned region2 v16+16 +;; @0062 store notrap aligned region5 v51, v26+64 +;; @0062 v52 = load.i64 notrap aligned region2 v16+24 +;; @0062 store notrap aligned region6 v52, v26+80 +;; v167 = iconst.i64 40 +;; v168 = iadd v28, v167 ; v167 = 40 +;; @0062 store notrap aligned region2 v162, v168+4 ; v162 = 1 +;; @0062 store.i64 notrap aligned region2 v56, v168+8 +;; v169 = iadd.i64 v0, v57 ; v57 = 48 +;; @0062 store notrap aligned region7 v169, v56 +;; @0062 store notrap aligned region2 v162, v168 ; v162 = 1 +;; @0062 store notrap aligned region2 v162, v28+56 ; v162 = 1 +;; v170 = iconst.i64 96 +;; v171 = iadd v25, v170 ; v170 = 96 +;; @0062 v67 = load.i64 notrap aligned region2 v171 +;; v172 = iconst.i64 -24 +;; v173 = iadd v67, v172 ; v172 = -24 +;; v174 = iconst.i64 0x0001_0000_0000 +;; @0062 v70 = stack_switch v173, v173, v174 ; v174 = 0x0001_0000_0000 +;; @0062 v72 = load.i64 notrap aligned region3 v26+88 +;; @0062 v73 = load.i64 notrap aligned region3 v26+96 +;; @0062 store notrap aligned region3 v27, v26+88 +;; @0062 store notrap aligned region3 v28, v26+96 +;; @0062 store notrap aligned region2 v162, v166 ; v162 = 1 +;; v175 = iconst.i32 0 +;; @0062 store notrap aligned region2 v175, v168 ; v175 = 0 +;; @0062 store notrap aligned region2 v175, v168+4 ; v175 = 0 +;; @0062 store notrap aligned region2 v160, v168+8 ; v160 = 0 +;; @0062 store notrap aligned region2 v160, v28+56 ; v160 = 0 +;; @0062 brif v70, block9, block6 ;; ;; block9: -;; v179 = iconst.i64 32 -;; v180 = ushr.i64 v69, v179 ; v179 = 32 -;; @0062 v82 = iconst.i64 4 -;; @0062 v83 = icmp eq v180, v82 ; v82 = 4 -;; @0062 brif v83, block8, block7 +;; v180 = iconst.i64 32 +;; v181 = ushr.i64 v70, v180 ; v180 = 32 +;; @0062 v83 = iconst.i64 4 +;; @0062 v84 = icmp eq v181, v83 ; v83 = 4 +;; @0062 brif v84, block8, block7 ;; ;; block8 cold: -;; @0062 v86 = iconst.i32 5 -;; v184 = iconst.i64 32 -;; v185 = iadd.i64 v72, v184 ; v184 = 32 -;; @0062 store notrap aligned region2 v86, v185 ; v86 = 5 -;; @0062 v91 = load.i64 notrap aligned region2 v27 -;; @0062 store notrap aligned region1 v91, v25+24 -;; @0062 v92 = load.i64 notrap aligned region2 v27+8 -;; @0062 store notrap aligned region4 v92, v25+72 -;; @0062 v93 = load.i64 notrap aligned region2 v27+16 -;; @0062 store notrap aligned region5 v93, v25+64 -;; @0062 v94 = load.i64 notrap aligned region2 v27+24 -;; @0062 store notrap aligned region6 v94, v25+80 -;; v186 = iconst.i32 0 -;; v187 = iconst.i64 120 -;; v188 = iadd.i64 v72, v187 ; v187 = 120 -;; @0062 store notrap aligned region2 v186, v188 ; v186 = 0 -;; @0062 store notrap aligned region2 v186, v188+4 ; v186 = 0 -;; v189 = iconst.i64 0 -;; @0062 store notrap aligned region2 v189, v188+8 ; v189 = 0 -;; v190 = iconst.i64 136 -;; v191 = iadd.i64 v72, v190 ; v190 = 136 -;; @0062 store notrap aligned region2 v186, v191 ; v186 = 0 -;; @0062 store notrap aligned region2 v186, v191+4 ; v186 = 0 -;; @0062 store notrap aligned region2 v189, v191+8 ; v189 = 0 +;; @0062 v87 = iconst.i32 5 +;; v185 = iconst.i64 32 +;; v186 = iadd.i64 v73, v185 ; v185 = 32 +;; @0062 store notrap aligned region2 v87, v186 ; v87 = 5 +;; @0062 v92 = load.i64 notrap aligned region2 v28 +;; @0062 store notrap aligned region1 v92, v26+24 +;; @0062 v93 = load.i64 notrap aligned region2 v28+8 +;; @0062 store notrap aligned region4 v93, v26+72 +;; @0062 v94 = load.i64 notrap aligned region2 v28+16 +;; @0062 store notrap aligned region5 v94, v26+64 +;; @0062 v95 = load.i64 notrap aligned region2 v28+24 +;; @0062 store notrap aligned region6 v95, v26+80 +;; v187 = iconst.i32 0 +;; v188 = iconst.i64 120 +;; v189 = iadd.i64 v73, v188 ; v188 = 120 +;; @0062 store notrap aligned region2 v187, v189 ; v187 = 0 +;; @0062 store notrap aligned region2 v187, v189+4 ; v187 = 0 +;; v190 = iconst.i64 0 +;; @0062 store notrap aligned region2 v190, v189+8 ; v190 = 0 +;; v191 = iconst.i64 144 +;; v192 = iadd.i64 v73, v191 ; v191 = 144 +;; @0062 store notrap aligned region2 v187, v192 ; v187 = 0 +;; @0062 store notrap aligned region2 v187, v192+4 ; v187 = 0 +;; @0062 store notrap aligned region2 v190, v192+8 ; v190 = 0 ;; @0062 try_call fn2(v0), sig2, block11, [ context v0 ] ;; ;; block11: ;; @0062 trap user12 ;; ;; block7: -;; @0062 v109 = load.i64 notrap aligned region4 v25+72 -;; @0062 v110 = load.i64 notrap aligned region5 v25+64 -;; @0062 v111 = load.i64 notrap aligned region6 v25+80 -;; @0062 store notrap aligned region2 v109, v72+8 -;; @0062 store notrap aligned region2 v110, v72+16 -;; @0062 store notrap aligned region2 v111, v72+24 -;; @0062 v114 = load.i64 notrap aligned region2 v27 -;; @0062 store notrap aligned region1 v114, v25+24 -;; @0062 v115 = load.i64 notrap aligned region2 v27+8 -;; @0062 store notrap aligned region4 v115, v25+72 -;; @0062 v116 = load.i64 notrap aligned region2 v27+16 -;; @0062 store notrap aligned region5 v116, v25+64 -;; @0062 v117 = load.i64 notrap aligned region2 v27+24 -;; @0062 store notrap aligned region6 v117, v25+80 -;; @0062 v119 = load.i64 notrap aligned region2 v72+88 +;; @0062 v110 = load.i64 notrap aligned region4 v26+72 +;; @0062 v111 = load.i64 notrap aligned region5 v26+64 +;; @0062 v112 = load.i64 notrap aligned region6 v26+80 +;; @0062 store notrap aligned region2 v110, v73+8 +;; @0062 store notrap aligned region2 v111, v73+16 +;; @0062 store notrap aligned region2 v112, v73+24 +;; @0062 v115 = load.i64 notrap aligned region2 v28 +;; @0062 store notrap aligned region1 v115, v26+24 +;; @0062 v116 = load.i64 notrap aligned region2 v28+8 +;; @0062 store notrap aligned region4 v116, v26+72 +;; @0062 v117 = load.i64 notrap aligned region2 v28+16 +;; @0062 store notrap aligned region5 v117, v26+64 +;; @0062 v118 = load.i64 notrap aligned region2 v28+24 +;; @0062 store notrap aligned region6 v118, v26+80 +;; @0062 v120 = load.i64 notrap aligned region2 v73+88 ;; @0062 jump block10 ;; ;; block12 cold: ;; @0062 trap user12 ;; ;; block13: -;; @0062 v126 = iconst.i64 136 -;; @0062 v127 = iadd.i64 v72, v126 ; v126 = 136 -;; @0062 v128 = load.i64 notrap aligned region2 v127+8 -;; @0062 v129 = load.i32 notrap aligned region7 v128 -;; v181 = iconst.i32 0 -;; @0062 store notrap aligned region2 v181, v127 ; v181 = 0 +;; @0062 v127 = iconst.i64 144 +;; @0062 v128 = iadd.i64 v73, v127 ; v127 = 144 +;; @0062 v129 = load.i64 notrap aligned region2 v128+8 +;; @0062 v130 = load.i32 notrap aligned region7 v129 +;; v182 = iconst.i32 0 +;; @0062 store notrap aligned region2 v182, v128 ; v182 = 0 ;; @0062 jump block4 ;; ;; block10: -;; @0062 v118 = ireduce.i32 v69 -;; @0062 br_table v118, block12, [block13] +;; @0062 v119 = ireduce.i32 v70 +;; @0062 br_table v119, block12, [block13] ;; ;; block6: -;; @0062 v133 = load.i64 notrap aligned region2 v27 -;; @0062 store notrap aligned region1 v133, v25+24 -;; @0062 v134 = load.i64 notrap aligned region2 v27+8 -;; @0062 store notrap aligned region4 v134, v25+72 -;; @0062 v135 = load.i64 notrap aligned region2 v27+16 -;; @0062 store notrap aligned region5 v135, v25+64 -;; @0062 v136 = load.i64 notrap aligned region2 v27+24 -;; @0062 store notrap aligned region6 v136, v25+80 -;; @0062 v139 = iconst.i32 4 -;; v175 = iconst.i64 32 -;; v176 = iadd.i64 v72, v175 ; v175 = 32 -;; @0062 store notrap aligned region2 v139, v176 ; v139 = 4 -;; @0062 v142 = iconst.i64 120 -;; @0062 v143 = iadd.i64 v72, v142 ; v142 = 120 -;; @0062 v144 = load.i64 notrap aligned region2 v143+8 -;; v177 = iconst.i32 0 -;; @0062 store notrap aligned region2 v177, v143 ; v177 = 0 -;; @0062 store notrap aligned region2 v177, v143+4 ; v177 = 0 -;; v178 = iconst.i64 0 -;; @0062 store notrap aligned region2 v178, v143+8 ; v178 = 0 +;; @0062 v134 = load.i64 notrap aligned region2 v28 +;; @0062 store notrap aligned region1 v134, v26+24 +;; @0062 v135 = load.i64 notrap aligned region2 v28+8 +;; @0062 store notrap aligned region4 v135, v26+72 +;; @0062 v136 = load.i64 notrap aligned region2 v28+16 +;; @0062 store notrap aligned region5 v136, v26+64 +;; @0062 v137 = load.i64 notrap aligned region2 v28+24 +;; @0062 store notrap aligned region6 v137, v26+80 +;; @0062 v140 = iconst.i32 4 +;; v176 = iconst.i64 32 +;; v177 = iadd.i64 v73, v176 ; v176 = 32 +;; @0062 store notrap aligned region2 v140, v177 ; v140 = 4 +;; @0062 v143 = iconst.i64 120 +;; @0062 v144 = iadd.i64 v73, v143 ; v143 = 120 +;; @0062 v145 = load.i64 notrap aligned region2 v144+8 +;; v178 = iconst.i32 0 +;; @0062 store notrap aligned region2 v178, v144 ; v178 = 0 +;; @0062 store notrap aligned region2 v178, v144+4 ; v178 = 0 +;; v179 = iconst.i64 0 +;; @0062 store notrap aligned region2 v179, v144+8 ; v179 = 0 ;; @0068 return ;; ;; block4: -;; @0062 v121 = uextend.i128 v119 -;; v182 = iconst.i64 64 -;; v183 = ishl v121, v182 ; v182 = 64 -;; @0062 v120 = uextend.i128 v72 -;; @0062 v125 = bor v183, v120 -;; @006d jump block2(v125) +;; @0062 v122 = uextend.i128 v120 +;; v183 = iconst.i64 64 +;; v184 = ishl v122, v183 ; v183 = 64 +;; @0062 v121 = uextend.i128 v73 +;; @0062 v126 = bor v184, v121 +;; @006d jump block2(v126) ;; } diff --git a/tests/disas/stack-switching/resume-suspend.wat b/tests/disas/stack-switching/resume-suspend.wat index b4960b22ea8a..5b289c3a6f56 100644 --- a/tests/disas/stack-switching/resume-suspend.wat +++ b/tests/disas/stack-switching/resume-suspend.wat @@ -84,8 +84,8 @@ ;; block6: ;; @003b store.i64 notrap aligned region3 v8, v6+80 ;; v84 = iconst.i32 1 -;; @003b v28 = iconst.i64 136 -;; @003b v29 = iadd.i64 v6, v28 ; v28 = 136 +;; @003b v28 = iconst.i64 144 +;; @003b v29 = iadd.i64 v6, v28 ; v28 = 144 ;; @003b store notrap aligned region3 v84, v29+4 ; v84 = 1 ;; @003b v31 = stack_addr.i64 ss0 ;; @003b store notrap aligned region3 v31, v29+8 @@ -111,7 +111,7 @@ ;; @003b brif v53, block8, block9 ;; ;; block8 cold: -;; v89 = iadd.i64 v6, v28 ; v28 = 136 +;; v89 = iadd.i64 v6, v28 ; v28 = 144 ;; @003b v56 = load.i64 notrap aligned region3 v89+8 ;; @003b v57 = load.i32 notrap aligned region4 v56 ;; v90 = iconst.i32 0 @@ -125,7 +125,7 @@ ;; @003b trap user12 ;; ;; block9: -;; v86 = iadd.i64 v6, v28 ; v28 = 136 +;; v86 = iadd.i64 v6, v28 ; v28 = 144 ;; @003b v63 = load.i64 notrap aligned region3 v86+8 ;; v87 = iconst.i32 0 ;; @003b store notrap aligned region3 v87, v86 ; v87 = 0 @@ -152,7 +152,7 @@ ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 ;; sig0 = (i64 vmctx, i32) -> i64 tail -;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail ;; sig2 = (i64 vmctx) tail ;; fn0 = colocated u805306368:6 sig0 ;; fn1 = colocated u805306368:42 sig1 @@ -163,193 +163,193 @@ ;; @0043 v9 = iconst.i32 0 ;; @0043 v10 = call fn0(v0, v9) ; v9 = 0 ;; @0045 trapz v10, user16 -;; @0045 v13 = call fn1(v0, v10, v9, v9) ; v9 = 0, v9 = 0 -;; @0045 v14 = load.i64 notrap aligned region2 v13+88 +;; @0045 v14 = call fn1(v0, v10, v9, v9, v9) ; v9 = 0, v9 = 0, v9 = 0 +;; @0045 v15 = load.i64 notrap aligned region2 v14+88 ;; @004e jump block3 ;; ;; block3: -;; @0045 v16 = uextend.i128 v14 +;; @0045 v17 = uextend.i128 v15 ;; @0040 v5 = iconst.i64 64 -;; v163 = ishl v16, v5 ; v5 = 64 -;; v165 = ireduce.i64 v163 -;; v167 = bor v165, v13 -;; @004e trapz v167, user16 -;; @004e v26 = load.i64 notrap aligned region2 v167+88 -;; @0045 v15 = uextend.i128 v13 -;; @0045 v20 = bor v163, v15 -;; v169 = ushr v20, v5 ; v5 = 64 -;; @004e v25 = ireduce.i64 v169 -;; @004e v27 = icmp eq v26, v25 -;; @004e trapz v27, user23 -;; @004e v28 = iconst.i64 1 -;; @004e v29 = iadd v26, v28 ; v28 = 1 -;; @004e store notrap aligned region2 v29, v167+88 -;; @004e v30 = load.i64 notrap aligned region2 v167+80 -;; @004e v31 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @004e v32 = load.i64 notrap aligned region3 v31+88 -;; @004e v33 = load.i64 notrap aligned region3 v31+96 -;; @004e store notrap aligned region2 v32, v30+64 -;; @004e store notrap aligned region2 v33, v30+72 +;; v164 = ishl v17, v5 ; v5 = 64 +;; v166 = ireduce.i64 v164 +;; v168 = bor v166, v14 +;; @004e trapz v168, user16 +;; @004e v27 = load.i64 notrap aligned region2 v168+88 +;; @0045 v16 = uextend.i128 v14 +;; @0045 v21 = bor v164, v16 +;; v170 = ushr v21, v5 ; v5 = 64 +;; @004e v26 = ireduce.i64 v170 +;; @004e v28 = icmp eq v27, v26 +;; @004e trapz v28, user23 +;; @004e v29 = iconst.i64 1 +;; @004e v30 = iadd v27, v29 ; v29 = 1 +;; @004e store notrap aligned region2 v30, v168+88 +;; @004e v31 = load.i64 notrap aligned region2 v168+80 +;; @004e v32 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @004e v33 = load.i64 notrap aligned region3 v32+88 +;; @004e v34 = load.i64 notrap aligned region3 v32+96 +;; @004e store notrap aligned region2 v33, v31+64 +;; @004e store notrap aligned region2 v34, v31+72 ;; @0040 v2 = iconst.i64 0 -;; @004e store notrap aligned region2 v2, v167+80 ; v2 = 0 -;; @004e v35 = iconst.i64 2 -;; @004e store notrap aligned region3 v35, v31+88 ; v35 = 2 -;; @004e store notrap aligned region3 v167, v31+96 -;; @004e v39 = iconst.i32 1 -;; @004e v40 = iconst.i64 32 -;; @004e v41 = iadd v167, v40 ; v40 = 32 -;; @004e store notrap aligned region2 v39, v41 ; v39 = 1 -;; @004e v42 = iconst.i32 2 -;; @004e v44 = iadd v33, v40 ; v40 = 32 -;; @004e store notrap aligned region2 v42, v44 ; v42 = 2 -;; @004e v48 = load.i64 notrap aligned region4 v31+72 -;; @004e v49 = load.i64 notrap aligned region5 v31+64 -;; @004e v50 = load.i64 notrap aligned region6 v31+80 -;; @004e store notrap aligned region2 v48, v33+8 -;; @004e store notrap aligned region2 v49, v33+16 -;; @004e store notrap aligned region2 v50, v33+24 -;; @004e v51 = load.i64 notrap aligned region1 v31+24 -;; @004e store notrap aligned region2 v51, v33 -;; @004e v54 = load.i64 notrap aligned region2 v167 -;; @004e store notrap aligned region1 v54, v31+24 -;; @004e v55 = load.i64 notrap aligned region2 v167+8 -;; @004e store notrap aligned region4 v55, v31+72 -;; @004e v56 = load.i64 notrap aligned region2 v167+16 -;; @004e store notrap aligned region5 v56, v31+64 -;; @004e v57 = load.i64 notrap aligned region2 v167+24 -;; @004e store notrap aligned region6 v57, v31+80 -;; @004e v58 = iconst.i64 40 -;; @004e v59 = iadd v33, v58 ; v58 = 40 -;; @004e store notrap aligned region2 v39, v59+4 ; v39 = 1 -;; @004e v61 = stack_addr.i64 ss0 -;; @004e store notrap aligned region2 v61, v59+8 -;; @004e v62 = iconst.i64 48 -;; @004e v63 = iadd.i64 v0, v62 ; v62 = 48 -;; @004e store notrap aligned region7 v63, v61 -;; @004e store notrap aligned region2 v39, v59 ; v39 = 1 -;; @004e store notrap aligned region2 v39, v33+56 ; v39 = 1 -;; @004e v70 = iconst.i64 96 -;; @004e v71 = iadd v30, v70 ; v70 = 96 -;; @004e v72 = load.i64 notrap aligned region2 v71 -;; @004e v73 = iconst.i64 -24 -;; @004e v74 = iadd v72, v73 ; v73 = -24 -;; v171 = iconst.i64 0x0001_0000_0000 -;; @004e v75 = stack_switch v74, v74, v171 ; v171 = 0x0001_0000_0000 -;; @004e v77 = load.i64 notrap aligned region3 v31+88 -;; @004e v78 = load.i64 notrap aligned region3 v31+96 -;; @004e store notrap aligned region3 v32, v31+88 -;; @004e store notrap aligned region3 v33, v31+96 -;; @004e store notrap aligned region2 v39, v44 ; v39 = 1 -;; v174 = iconst.i32 0 -;; @004e store notrap aligned region2 v174, v59 ; v174 = 0 -;; @004e store notrap aligned region2 v174, v59+4 ; v174 = 0 -;; @004e store notrap aligned region2 v2, v59+8 ; v2 = 0 -;; @004e store notrap aligned region2 v2, v33+56 ; v2 = 0 -;; @004e brif v75, block7, block4 +;; @004e store notrap aligned region2 v2, v168+80 ; v2 = 0 +;; @004e v36 = iconst.i64 2 +;; @004e store notrap aligned region3 v36, v32+88 ; v36 = 2 +;; @004e store notrap aligned region3 v168, v32+96 +;; @004e v40 = iconst.i32 1 +;; @004e v41 = iconst.i64 32 +;; @004e v42 = iadd v168, v41 ; v41 = 32 +;; @004e store notrap aligned region2 v40, v42 ; v40 = 1 +;; @004e v43 = iconst.i32 2 +;; @004e v45 = iadd v34, v41 ; v41 = 32 +;; @004e store notrap aligned region2 v43, v45 ; v43 = 2 +;; @004e v49 = load.i64 notrap aligned region4 v32+72 +;; @004e v50 = load.i64 notrap aligned region5 v32+64 +;; @004e v51 = load.i64 notrap aligned region6 v32+80 +;; @004e store notrap aligned region2 v49, v34+8 +;; @004e store notrap aligned region2 v50, v34+16 +;; @004e store notrap aligned region2 v51, v34+24 +;; @004e v52 = load.i64 notrap aligned region1 v32+24 +;; @004e store notrap aligned region2 v52, v34 +;; @004e v55 = load.i64 notrap aligned region2 v168 +;; @004e store notrap aligned region1 v55, v32+24 +;; @004e v56 = load.i64 notrap aligned region2 v168+8 +;; @004e store notrap aligned region4 v56, v32+72 +;; @004e v57 = load.i64 notrap aligned region2 v168+16 +;; @004e store notrap aligned region5 v57, v32+64 +;; @004e v58 = load.i64 notrap aligned region2 v168+24 +;; @004e store notrap aligned region6 v58, v32+80 +;; @004e v59 = iconst.i64 40 +;; @004e v60 = iadd v34, v59 ; v59 = 40 +;; @004e store notrap aligned region2 v40, v60+4 ; v40 = 1 +;; @004e v62 = stack_addr.i64 ss0 +;; @004e store notrap aligned region2 v62, v60+8 +;; @004e v63 = iconst.i64 48 +;; @004e v64 = iadd.i64 v0, v63 ; v63 = 48 +;; @004e store notrap aligned region7 v64, v62 +;; @004e store notrap aligned region2 v40, v60 ; v40 = 1 +;; @004e store notrap aligned region2 v40, v34+56 ; v40 = 1 +;; @004e v71 = iconst.i64 96 +;; @004e v72 = iadd v31, v71 ; v71 = 96 +;; @004e v73 = load.i64 notrap aligned region2 v72 +;; @004e v74 = iconst.i64 -24 +;; @004e v75 = iadd v73, v74 ; v74 = -24 +;; v172 = iconst.i64 0x0001_0000_0000 +;; @004e v76 = stack_switch v75, v75, v172 ; v172 = 0x0001_0000_0000 +;; @004e v78 = load.i64 notrap aligned region3 v32+88 +;; @004e v79 = load.i64 notrap aligned region3 v32+96 +;; @004e store notrap aligned region3 v33, v32+88 +;; @004e store notrap aligned region3 v34, v32+96 +;; @004e store notrap aligned region2 v40, v45 ; v40 = 1 +;; v175 = iconst.i32 0 +;; @004e store notrap aligned region2 v175, v60 ; v175 = 0 +;; @004e store notrap aligned region2 v175, v60+4 ; v175 = 0 +;; @004e store notrap aligned region2 v2, v60+8 ; v2 = 0 +;; @004e store notrap aligned region2 v2, v34+56 ; v2 = 0 +;; @004e brif v76, block7, block4 ;; ;; block7: -;; v182 = iconst.i64 32 -;; v183 = ushr.i64 v75, v182 ; v182 = 32 -;; @004e v88 = iconst.i64 4 -;; @004e v89 = icmp eq v183, v88 ; v88 = 4 -;; @004e brif v89, block6, block5 +;; v183 = iconst.i64 32 +;; v184 = ushr.i64 v76, v183 ; v183 = 32 +;; @004e v89 = iconst.i64 4 +;; @004e v90 = icmp eq v184, v89 ; v89 = 4 +;; @004e brif v90, block6, block5 ;; ;; block6 cold: -;; @004e v92 = iconst.i32 5 -;; v187 = iconst.i64 32 -;; v188 = iadd.i64 v78, v187 ; v187 = 32 -;; @004e store notrap aligned region2 v92, v188 ; v92 = 5 -;; @004e v97 = load.i64 notrap aligned region2 v33 -;; @004e store notrap aligned region1 v97, v31+24 -;; @004e v98 = load.i64 notrap aligned region2 v33+8 -;; @004e store notrap aligned region4 v98, v31+72 -;; @004e v99 = load.i64 notrap aligned region2 v33+16 -;; @004e store notrap aligned region5 v99, v31+64 -;; @004e v100 = load.i64 notrap aligned region2 v33+24 -;; @004e store notrap aligned region6 v100, v31+80 -;; v189 = iconst.i32 0 -;; v190 = iconst.i64 120 -;; v191 = iadd.i64 v78, v190 ; v190 = 120 -;; @004e store notrap aligned region2 v189, v191 ; v189 = 0 -;; @004e store notrap aligned region2 v189, v191+4 ; v189 = 0 -;; v192 = iconst.i64 0 -;; @004e store notrap aligned region2 v192, v191+8 ; v192 = 0 -;; v193 = iconst.i64 136 -;; v194 = iadd.i64 v78, v193 ; v193 = 136 -;; @004e store notrap aligned region2 v189, v194 ; v189 = 0 -;; @004e store notrap aligned region2 v189, v194+4 ; v189 = 0 -;; @004e store notrap aligned region2 v192, v194+8 ; v192 = 0 +;; @004e v93 = iconst.i32 5 +;; v188 = iconst.i64 32 +;; v189 = iadd.i64 v79, v188 ; v188 = 32 +;; @004e store notrap aligned region2 v93, v189 ; v93 = 5 +;; @004e v98 = load.i64 notrap aligned region2 v34 +;; @004e store notrap aligned region1 v98, v32+24 +;; @004e v99 = load.i64 notrap aligned region2 v34+8 +;; @004e store notrap aligned region4 v99, v32+72 +;; @004e v100 = load.i64 notrap aligned region2 v34+16 +;; @004e store notrap aligned region5 v100, v32+64 +;; @004e v101 = load.i64 notrap aligned region2 v34+24 +;; @004e store notrap aligned region6 v101, v32+80 +;; v190 = iconst.i32 0 +;; v191 = iconst.i64 120 +;; v192 = iadd.i64 v79, v191 ; v191 = 120 +;; @004e store notrap aligned region2 v190, v192 ; v190 = 0 +;; @004e store notrap aligned region2 v190, v192+4 ; v190 = 0 +;; v193 = iconst.i64 0 +;; @004e store notrap aligned region2 v193, v192+8 ; v193 = 0 +;; v194 = iconst.i64 144 +;; v195 = iadd.i64 v79, v194 ; v194 = 144 +;; @004e store notrap aligned region2 v190, v195 ; v190 = 0 +;; @004e store notrap aligned region2 v190, v195+4 ; v190 = 0 +;; @004e store notrap aligned region2 v193, v195+8 ; v193 = 0 ;; @004e try_call fn2(v0), sig2, block9, [ context v0 ] ;; ;; block9: ;; @004e trap user12 ;; ;; block5: -;; @004e v115 = load.i64 notrap aligned region4 v31+72 -;; @004e v116 = load.i64 notrap aligned region5 v31+64 -;; @004e v117 = load.i64 notrap aligned region6 v31+80 -;; @004e store notrap aligned region2 v115, v78+8 -;; @004e store notrap aligned region2 v116, v78+16 -;; @004e store notrap aligned region2 v117, v78+24 -;; @004e v120 = load.i64 notrap aligned region2 v33 -;; @004e store notrap aligned region1 v120, v31+24 -;; @004e v121 = load.i64 notrap aligned region2 v33+8 -;; @004e store notrap aligned region4 v121, v31+72 -;; @004e v122 = load.i64 notrap aligned region2 v33+16 -;; @004e store notrap aligned region5 v122, v31+64 -;; @004e v123 = load.i64 notrap aligned region2 v33+24 -;; @004e store notrap aligned region6 v123, v31+80 -;; @004e v125 = load.i64 notrap aligned region2 v78+88 +;; @004e v116 = load.i64 notrap aligned region4 v32+72 +;; @004e v117 = load.i64 notrap aligned region5 v32+64 +;; @004e v118 = load.i64 notrap aligned region6 v32+80 +;; @004e store notrap aligned region2 v116, v79+8 +;; @004e store notrap aligned region2 v117, v79+16 +;; @004e store notrap aligned region2 v118, v79+24 +;; @004e v121 = load.i64 notrap aligned region2 v34 +;; @004e store notrap aligned region1 v121, v32+24 +;; @004e v122 = load.i64 notrap aligned region2 v34+8 +;; @004e store notrap aligned region4 v122, v32+72 +;; @004e v123 = load.i64 notrap aligned region2 v34+16 +;; @004e store notrap aligned region5 v123, v32+64 +;; @004e v124 = load.i64 notrap aligned region2 v34+24 +;; @004e store notrap aligned region6 v124, v32+80 +;; @004e v126 = load.i64 notrap aligned region2 v79+88 ;; @004e jump block8 ;; ;; block10 cold: ;; @004e trap user12 ;; ;; block11: -;; @004e v132 = iconst.i64 136 -;; @004e v133 = iadd.i64 v78, v132 ; v132 = 136 -;; @004e v134 = load.i64 notrap aligned region2 v133+8 -;; v184 = iconst.i32 0 -;; @004e store notrap aligned region2 v184, v133 ; v184 = 0 -;; @004e v127 = uextend.i128 v125 -;; v185 = iconst.i64 64 -;; v186 = ishl v127, v185 ; v185 = 64 -;; @004e v126 = uextend.i128 v78 -;; @004e v131 = bor v186, v126 -;; @004e jump block2(v131) +;; @004e v133 = iconst.i64 144 +;; @004e v134 = iadd.i64 v79, v133 ; v133 = 144 +;; @004e v135 = load.i64 notrap aligned region2 v134+8 +;; v185 = iconst.i32 0 +;; @004e store notrap aligned region2 v185, v134 ; v185 = 0 +;; @004e v128 = uextend.i128 v126 +;; v186 = iconst.i64 64 +;; v187 = ishl v128, v186 ; v186 = 64 +;; @004e v127 = uextend.i128 v79 +;; @004e v132 = bor v187, v127 +;; @004e jump block2(v132) ;; ;; block8: -;; @004e v124 = ireduce.i32 v75 -;; @004e br_table v124, block10, [block11] +;; @004e v125 = ireduce.i32 v76 +;; @004e br_table v125, block10, [block11] ;; ;; block4: -;; @004e v138 = load.i64 notrap aligned region2 v33 -;; @004e store notrap aligned region1 v138, v31+24 -;; @004e v139 = load.i64 notrap aligned region2 v33+8 -;; @004e store notrap aligned region4 v139, v31+72 -;; @004e v140 = load.i64 notrap aligned region2 v33+16 -;; @004e store notrap aligned region5 v140, v31+64 -;; @004e v141 = load.i64 notrap aligned region2 v33+24 -;; @004e store notrap aligned region6 v141, v31+80 -;; @004e v144 = iconst.i32 4 -;; v175 = iconst.i64 32 -;; v176 = iadd.i64 v78, v175 ; v175 = 32 -;; @004e store notrap aligned region2 v144, v176 ; v144 = 4 -;; @004e v147 = iconst.i64 120 -;; @004e v148 = iadd.i64 v78, v147 ; v147 = 120 -;; @004e v149 = load.i64 notrap aligned region2 v148+8 -;; v177 = iconst.i32 0 -;; @004e store notrap aligned region2 v177, v148 ; v177 = 0 -;; @004e store notrap aligned region2 v177, v148+4 ; v177 = 0 -;; v178 = iconst.i64 0 -;; @004e store notrap aligned region2 v178, v148+8 ; v178 = 0 -;; v179 = uextend.i128 v178 ; v178 = 0 -;; v180 = iconst.i64 64 -;; v181 = ishl v179, v180 ; v180 = 64 -;; @0040 v8 = bor v181, v179 +;; @004e v139 = load.i64 notrap aligned region2 v34 +;; @004e store notrap aligned region1 v139, v32+24 +;; @004e v140 = load.i64 notrap aligned region2 v34+8 +;; @004e store notrap aligned region4 v140, v32+72 +;; @004e v141 = load.i64 notrap aligned region2 v34+16 +;; @004e store notrap aligned region5 v141, v32+64 +;; @004e v142 = load.i64 notrap aligned region2 v34+24 +;; @004e store notrap aligned region6 v142, v32+80 +;; @004e v145 = iconst.i32 4 +;; v176 = iconst.i64 32 +;; v177 = iadd.i64 v79, v176 ; v176 = 32 +;; @004e store notrap aligned region2 v145, v177 ; v145 = 4 +;; @004e v148 = iconst.i64 120 +;; @004e v149 = iadd.i64 v79, v148 ; v148 = 120 +;; @004e v150 = load.i64 notrap aligned region2 v149+8 +;; v178 = iconst.i32 0 +;; @004e store notrap aligned region2 v178, v149 ; v178 = 0 +;; @004e store notrap aligned region2 v178, v149+4 ; v178 = 0 +;; v179 = iconst.i64 0 +;; @004e store notrap aligned region2 v179, v149+8 ; v179 = 0 +;; v180 = uextend.i128 v179 ; v179 = 0 +;; v181 = iconst.i64 64 +;; v182 = ishl v180, v181 ; v181 = 64 +;; @0040 v8 = bor v182, v180 ;; @0056 jump block2(v8) ;; -;; block2(v160: i128): +;; block2(v161: i128): ;; @0058 jump block1 ;; ;; block1: diff --git a/tests/disas/stack-switching/symmetric-switch.wat b/tests/disas/stack-switching/symmetric-switch.wat index a88086bbc5c0..8ab86b4efc07 100644 --- a/tests/disas/stack-switching/symmetric-switch.wat +++ b/tests/disas/stack-switching/symmetric-switch.wat @@ -41,7 +41,7 @@ ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 ;; sig0 = (i64 vmctx, i32) -> i64 tail -;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail ;; sig2 = (i64 vmctx, i32) -> i8 tail ;; fn0 = colocated u805306368:6 sig0 ;; fn1 = colocated u805306368:42 sig1 @@ -54,221 +54,222 @@ ;; @003c trapz v3, user16 ;; @003c v4 = iconst.i32 1 ;; @003c v5 = iconst.i32 0 -;; @003c v6 = call fn1(v0, v3, v4, v5) ; v4 = 1, v5 = 0 -;; @003c v7 = load.i64 notrap aligned region2 v6+88 -;; @003c v8 = uextend.i128 v6 +;; @003c v6 = iconst.i32 0 +;; @003c v7 = call fn1(v0, v3, v4, v5, v6) ; v4 = 1, v5 = 0, v6 = 0 +;; @003c v8 = load.i64 notrap aligned region2 v7+88 ;; @003c v9 = uextend.i128 v7 -;; @003c v10 = iconst.i64 64 -;; @003c v11 = uextend.i128 v10 ; v10 = 64 -;; @003c v12 = ishl v9, v11 -;; @003c v13 = bor v12, v8 -;; @003e v14 = ireduce.i64 v13 -;; @003e v15 = iconst.i64 64 -;; @003e v16 = uextend.i128 v15 ; v15 = 64 -;; @003e v17 = ushr v13, v16 -;; @003e v18 = ireduce.i64 v17 -;; @003e trapz v14, user16 -;; @003e v19 = load.i64 notrap aligned region2 v14+88 -;; @003e v20 = icmp eq v19, v18 -;; @003e trapz v20, user23 -;; @003e v21 = iconst.i64 1 -;; @003e v22 = iadd v19, v21 ; v21 = 1 -;; @003e store notrap aligned region2 v22, v14+88 -;; @003e v23 = iconst.i64 48 -;; @003e v24 = iadd v0, v23 ; v23 = 48 -;; @003e v25 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @003e v26 = load.i64 notrap aligned region3 v25+88 -;; @003e v27 = load.i64 notrap aligned region3 v25+96 -;; @003e jump block2(v26, v27) +;; @003c v10 = uextend.i128 v8 +;; @003c v11 = iconst.i64 64 +;; @003c v12 = uextend.i128 v11 ; v11 = 64 +;; @003c v13 = ishl v10, v12 +;; @003c v14 = bor v13, v9 +;; @003e v15 = ireduce.i64 v14 +;; @003e v16 = iconst.i64 64 +;; @003e v17 = uextend.i128 v16 ; v16 = 64 +;; @003e v18 = ushr v14, v17 +;; @003e v19 = ireduce.i64 v18 +;; @003e trapz v15, user16 +;; @003e v20 = load.i64 notrap aligned region2 v15+88 +;; @003e v21 = icmp eq v20, v19 +;; @003e trapz v21, user23 +;; @003e v22 = iconst.i64 1 +;; @003e v23 = iadd v20, v22 ; v22 = 1 +;; @003e store notrap aligned region2 v23, v15+88 +;; @003e v24 = iconst.i64 48 +;; @003e v25 = iadd v0, v24 ; v24 = 48 +;; @003e v26 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @003e v27 = load.i64 notrap aligned region3 v26+88 +;; @003e v28 = load.i64 notrap aligned region3 v26+96 +;; @003e jump block2(v27, v28) ;; -;; block2(v28: i64, v29: i64): -;; @003e v30 = iconst.i64 1 -;; @003e v31 = icmp eq v28, v30 ; v30 = 1 -;; @003e brif v31, block7, block3 +;; block2(v29: i64, v30: i64): +;; @003e v31 = iconst.i64 1 +;; @003e v32 = icmp eq v29, v31 ; v31 = 1 +;; @003e brif v32, block7, block3 ;; ;; block3: -;; @003e v32 = load.i64 notrap aligned region2 v29+64 -;; @003e v33 = load.i64 notrap aligned region2 v29+72 -;; @003e v34 = iconst.i64 40 -;; @003e v35 = iadd v33, v34 ; v34 = 40 -;; @003e v36 = load.i64 notrap aligned region2 v35+8 -;; @003e v37 = load.i32 notrap aligned region2 v33+56 -;; @003e v38 = load.i32 notrap aligned region2 v35 -;; @003e jump block4(v37) +;; @003e v33 = load.i64 notrap aligned region2 v30+64 +;; @003e v34 = load.i64 notrap aligned region2 v30+72 +;; @003e v35 = iconst.i64 40 +;; @003e v36 = iadd v34, v35 ; v35 = 40 +;; @003e v37 = load.i64 notrap aligned region2 v36+8 +;; @003e v38 = load.i32 notrap aligned region2 v34+56 +;; @003e v39 = load.i32 notrap aligned region2 v36 +;; @003e jump block4(v38) ;; -;; block4(v39: i32): -;; @003e v40 = icmp ult v39, v38 -;; @003e brif v40, block5, block2(v32, v33) +;; block4(v40: i32): +;; @003e v41 = icmp ult v40, v39 +;; @003e brif v41, block5, block2(v33, v34) ;; ;; block5: -;; @003e v41 = iconst.i32 8 -;; @003e v42 = imul.i32 v39, v41 ; v41 = 8 -;; @003e v43 = uextend.i64 v42 -;; @003e v44 = iadd.i64 v36, v43 -;; @003e v45 = load.i64 notrap aligned region4 v44 -;; @003e v46 = icmp eq v45, v24 -;; @003e v47 = iconst.i32 1 -;; @003e v48 = iadd.i32 v39, v47 ; v47 = 1 -;; @003e brif v46, block6, block4(v48) +;; @003e v42 = iconst.i32 8 +;; @003e v43 = imul.i32 v40, v42 ; v42 = 8 +;; @003e v44 = uextend.i64 v43 +;; @003e v45 = iadd.i64 v37, v44 +;; @003e v46 = load.i64 notrap aligned region4 v45 +;; @003e v47 = icmp eq v46, v25 +;; @003e v48 = iconst.i32 1 +;; @003e v49 = iadd.i32 v40, v48 ; v48 = 1 +;; @003e brif v47, block6, block4(v49) ;; ;; block7 cold: ;; @003e trap user22 ;; ;; block6: -;; @003e store.i64 notrap aligned region2 v29, v27+80 -;; @003e v49 = iconst.i64 136 -;; @003e v50 = iadd.i64 v27, v49 ; v49 = 136 -;; @003e v51 = iconst.i32 1 -;; @003e v52 = stack_addr.i64 ss0 -;; @003e store notrap aligned region2 v51, v50+4 ; v51 = 1 -;; @003e store notrap aligned region2 v52, v50+8 -;; @003e v53 = iconst.i64 0 -;; @003e v54 = iadd.i64 v27, v53 ; v53 = 0 -;; @003e v55 = iconst.i32 3 -;; @003e v56 = iconst.i64 32 -;; @003e v57 = iadd v54, v56 ; v56 = 32 -;; @003e store notrap aligned region2 v55, v57 ; v55 = 3 -;; @003e v58 = iconst.i64 0 +;; @003e store.i64 notrap aligned region2 v30, v28+80 +;; @003e v50 = iconst.i64 144 +;; @003e v51 = iadd.i64 v28, v50 ; v50 = 144 +;; @003e v52 = iconst.i32 1 +;; @003e v53 = stack_addr.i64 ss0 +;; @003e store notrap aligned region2 v52, v51+4 ; v52 = 1 +;; @003e store notrap aligned region2 v53, v51+8 +;; @003e v54 = iconst.i64 0 +;; @003e v55 = iadd.i64 v28, v54 ; v54 = 0 +;; @003e v56 = iconst.i32 3 +;; @003e v57 = iconst.i64 32 +;; @003e v58 = iadd v55, v57 ; v57 = 32 +;; @003e store notrap aligned region2 v56, v58 ; v56 = 3 ;; @003e v59 = iconst.i64 0 -;; @003e store notrap aligned region2 v58, v29+64 ; v58 = 0 -;; @003e store notrap aligned region2 v59, v29+72 ; v59 = 0 -;; @003e v60 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @003e v61 = iconst.i64 0 -;; @003e v62 = iadd v54, v61 ; v61 = 0 -;; @003e v63 = load.i64 notrap aligned region5 v60+72 -;; @003e v64 = load.i64 notrap aligned region6 v60+64 -;; @003e v65 = load.i64 notrap aligned region7 v60+80 -;; @003e store notrap aligned region2 v63, v62+8 -;; @003e store notrap aligned region2 v64, v62+16 -;; @003e store notrap aligned region2 v65, v62+24 -;; @003e v66 = load.i64 notrap aligned region2 v27+88 -;; @003e v67 = uextend.i128 v27 -;; @003e v68 = uextend.i128 v66 -;; @003e v69 = iconst.i64 64 -;; @003e v70 = uextend.i128 v69 ; v69 = 64 -;; @003e v71 = ishl v68, v70 -;; @003e v72 = bor v71, v67 -;; @003e v74 = iconst.i64 0 -;; @003e v75 = iadd.i64 v14, v74 ; v74 = 0 -;; @003e v76 = iconst.i64 32 -;; @003e v77 = iadd v75, v76 ; v76 = 32 -;; @003e v78 = load.i32 notrap aligned region2 v77 -;; @003e v79 = iconst.i32 0 -;; @003e v80 = icmp ne v78, v79 ; v79 = 0 -;; @003e brif v80, block9, block8 +;; @003e v60 = iconst.i64 0 +;; @003e store notrap aligned region2 v59, v30+64 ; v59 = 0 +;; @003e store notrap aligned region2 v60, v30+72 ; v60 = 0 +;; @003e v61 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @003e v62 = iconst.i64 0 +;; @003e v63 = iadd v55, v62 ; v62 = 0 +;; @003e v64 = load.i64 notrap aligned region5 v61+72 +;; @003e v65 = load.i64 notrap aligned region6 v61+64 +;; @003e v66 = load.i64 notrap aligned region7 v61+80 +;; @003e store notrap aligned region2 v64, v63+8 +;; @003e store notrap aligned region2 v65, v63+16 +;; @003e store notrap aligned region2 v66, v63+24 +;; @003e v67 = load.i64 notrap aligned region2 v28+88 +;; @003e v68 = uextend.i128 v28 +;; @003e v69 = uextend.i128 v67 +;; @003e v70 = iconst.i64 64 +;; @003e v71 = uextend.i128 v70 ; v70 = 64 +;; @003e v72 = ishl v69, v71 +;; @003e v73 = bor v72, v68 +;; @003e v75 = iconst.i64 0 +;; @003e v76 = iadd.i64 v15, v75 ; v75 = 0 +;; @003e v77 = iconst.i64 32 +;; @003e v78 = iadd v76, v77 ; v77 = 32 +;; @003e v79 = load.i32 notrap aligned region2 v78 +;; @003e v80 = iconst.i32 0 +;; @003e v81 = icmp ne v79, v80 ; v80 = 0 +;; @003e brif v81, block9, block8 ;; ;; block8: -;; @003e v81 = iconst.i64 120 -;; @003e v82 = iadd.i64 v14, v81 ; v81 = 120 -;; @003e v83 = load.i64 notrap aligned region2 v82+8 -;; @003e v84 = load.i32 notrap aligned region2 v82 -;; @003e v85 = iconst.i32 1 -;; @003e v86 = iadd v84, v85 ; v85 = 1 -;; @003e store notrap aligned region2 v86, v82 -;; @003e v87 = uextend.i64 v84 -;; @003e v88 = iconst.i64 16 -;; @003e v89 = imul v87, v88 ; v88 = 16 -;; @003e v90 = iadd v83, v89 -;; @003e jump block10(v90) +;; @003e v82 = iconst.i64 120 +;; @003e v83 = iadd.i64 v15, v82 ; v82 = 120 +;; @003e v84 = load.i64 notrap aligned region2 v83+8 +;; @003e v85 = load.i32 notrap aligned region2 v83 +;; @003e v86 = iconst.i32 1 +;; @003e v87 = iadd v85, v86 ; v86 = 1 +;; @003e store notrap aligned region2 v87, v83 +;; @003e v88 = uextend.i64 v85 +;; @003e v89 = iconst.i64 16 +;; @003e v90 = imul v88, v89 ; v89 = 16 +;; @003e v91 = iadd v84, v90 +;; @003e jump block10(v91) ;; ;; block9: -;; @003e v91 = iconst.i64 136 -;; @003e v92 = iadd.i64 v14, v91 ; v91 = 136 -;; @003e v93 = load.i64 notrap aligned region2 v92+8 -;; @003e v94 = load.i32 notrap aligned region2 v92 -;; @003e v95 = iconst.i32 1 -;; @003e v96 = iadd v94, v95 ; v95 = 1 -;; @003e store notrap aligned region2 v96, v92 -;; @003e v97 = uextend.i64 v94 -;; @003e v98 = iconst.i64 16 -;; @003e v99 = imul v97, v98 ; v98 = 16 -;; @003e v100 = iadd v93, v99 -;; @003e jump block10(v100) +;; @003e v92 = iconst.i64 144 +;; @003e v93 = iadd.i64 v15, v92 ; v92 = 144 +;; @003e v94 = load.i64 notrap aligned region2 v93+8 +;; @003e v95 = load.i32 notrap aligned region2 v93 +;; @003e v96 = iconst.i32 1 +;; @003e v97 = iadd v95, v96 ; v96 = 1 +;; @003e store notrap aligned region2 v97, v93 +;; @003e v98 = uextend.i64 v95 +;; @003e v99 = iconst.i64 16 +;; @003e v100 = imul v98, v99 ; v99 = 16 +;; @003e v101 = iadd v94, v100 +;; @003e jump block10(v101) ;; -;; block10(v73: i64): -;; @003e store.i128 notrap aligned region4 v72, v73 -;; @003e v101 = iconst.i64 0 -;; @003e v102 = iadd.i64 v14, v101 ; v101 = 0 -;; @003e v103 = iconst.i32 1 -;; @003e v104 = iconst.i64 32 -;; @003e v105 = iadd v102, v104 ; v104 = 32 -;; @003e store notrap aligned region2 v103, v105 ; v103 = 1 -;; @003e v106 = load.i64 notrap aligned region2 v14+80 -;; @003e store.i64 notrap aligned region2 v32, v106+64 -;; @003e store.i64 notrap aligned region2 v33, v106+72 -;; @003e v107 = iconst.i64 2 -;; @003e v108 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @003e store notrap aligned region3 v107, v108+88 ; v107 = 2 -;; @003e store.i64 notrap aligned region3 v14, v108+96 -;; @003e v109 = iconst.i64 0 -;; @003e v110 = iadd v102, v109 ; v109 = 0 -;; @003e v111 = load.i64 notrap aligned region2 v110 -;; @003e store notrap aligned region1 v111, v60+24 -;; @003e v112 = load.i64 notrap aligned region2 v110+8 -;; @003e store notrap aligned region5 v112, v60+72 -;; @003e v113 = load.i64 notrap aligned region2 v110+16 -;; @003e store notrap aligned region6 v113, v60+64 -;; @003e v114 = load.i64 notrap aligned region2 v110+24 -;; @003e store notrap aligned region7 v114, v60+80 -;; @003e v115 = iconst.i64 96 -;; @003e v116 = iadd.i64 v29, v115 ; v115 = 96 -;; @003e v117 = load.i64 notrap aligned region2 v116 -;; @003e v118 = iconst.i64 -24 -;; @003e v119 = iadd v117, v118 ; v118 = -24 -;; @003e v120 = iconst.i64 96 -;; @003e v121 = iadd v106, v120 ; v120 = 96 -;; @003e v122 = load.i64 notrap aligned region2 v121 -;; @003e v123 = iconst.i64 -24 -;; @003e v124 = iadd v122, v123 ; v123 = -24 -;; @003e v125 = stack_addr.i64 ss1 -;; @003e v126 = load.i64 notrap aligned region4 v124 -;; @003e store notrap aligned region8 v126, v125 -;; @003e v127 = load.i64 notrap aligned region4 v119 -;; @003e store notrap aligned region4 v127, v124 -;; @003e v128 = load.i64 notrap aligned region4 v124+8 -;; @003e store notrap aligned region8 v128, v125+8 -;; @003e v129 = load.i64 notrap aligned region4 v119+8 -;; @003e store notrap aligned region4 v129, v124+8 -;; @003e v130 = load.i64 notrap aligned region4 v124+16 -;; @003e store notrap aligned region8 v130, v125+16 -;; @003e v131 = load.i64 notrap aligned region4 v119+16 -;; @003e store notrap aligned region4 v131, v124+16 -;; @003e v132 = iconst.i64 3 -;; @003e v133 = iconst.i64 32 -;; @003e v134 = ishl v132, v133 ; v132 = 3, v133 = 32 -;; @003e v135 = stack_switch v119, v125, v134 -;; @003e v136 = iconst.i64 32 -;; @003e v137 = ushr v135, v136 ; v136 = 32 -;; @003e v138 = iconst.i64 5 -;; @003e v139 = icmp eq v137, v138 ; v138 = 5 -;; @003e brif v139, block11, block12 +;; block10(v74: i64): +;; @003e store.i128 notrap aligned region4 v73, v74 +;; @003e v102 = iconst.i64 0 +;; @003e v103 = iadd.i64 v15, v102 ; v102 = 0 +;; @003e v104 = iconst.i32 1 +;; @003e v105 = iconst.i64 32 +;; @003e v106 = iadd v103, v105 ; v105 = 32 +;; @003e store notrap aligned region2 v104, v106 ; v104 = 1 +;; @003e v107 = load.i64 notrap aligned region2 v15+80 +;; @003e store.i64 notrap aligned region2 v33, v107+64 +;; @003e store.i64 notrap aligned region2 v34, v107+72 +;; @003e v108 = iconst.i64 2 +;; @003e v109 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @003e store notrap aligned region3 v108, v109+88 ; v108 = 2 +;; @003e store.i64 notrap aligned region3 v15, v109+96 +;; @003e v110 = iconst.i64 0 +;; @003e v111 = iadd v103, v110 ; v110 = 0 +;; @003e v112 = load.i64 notrap aligned region2 v111 +;; @003e store notrap aligned region1 v112, v61+24 +;; @003e v113 = load.i64 notrap aligned region2 v111+8 +;; @003e store notrap aligned region5 v113, v61+72 +;; @003e v114 = load.i64 notrap aligned region2 v111+16 +;; @003e store notrap aligned region6 v114, v61+64 +;; @003e v115 = load.i64 notrap aligned region2 v111+24 +;; @003e store notrap aligned region7 v115, v61+80 +;; @003e v116 = iconst.i64 96 +;; @003e v117 = iadd.i64 v30, v116 ; v116 = 96 +;; @003e v118 = load.i64 notrap aligned region2 v117 +;; @003e v119 = iconst.i64 -24 +;; @003e v120 = iadd v118, v119 ; v119 = -24 +;; @003e v121 = iconst.i64 96 +;; @003e v122 = iadd v107, v121 ; v121 = 96 +;; @003e v123 = load.i64 notrap aligned region2 v122 +;; @003e v124 = iconst.i64 -24 +;; @003e v125 = iadd v123, v124 ; v124 = -24 +;; @003e v126 = stack_addr.i64 ss1 +;; @003e v127 = load.i64 notrap aligned region4 v125 +;; @003e store notrap aligned region8 v127, v126 +;; @003e v128 = load.i64 notrap aligned region4 v120 +;; @003e store notrap aligned region4 v128, v125 +;; @003e v129 = load.i64 notrap aligned region4 v125+8 +;; @003e store notrap aligned region8 v129, v126+8 +;; @003e v130 = load.i64 notrap aligned region4 v120+8 +;; @003e store notrap aligned region4 v130, v125+8 +;; @003e v131 = load.i64 notrap aligned region4 v125+16 +;; @003e store notrap aligned region8 v131, v126+16 +;; @003e v132 = load.i64 notrap aligned region4 v120+16 +;; @003e store notrap aligned region4 v132, v125+16 +;; @003e v133 = iconst.i64 3 +;; @003e v134 = iconst.i64 32 +;; @003e v135 = ishl v133, v134 ; v133 = 3, v134 = 32 +;; @003e v136 = stack_switch v120, v126, v135 +;; @003e v137 = iconst.i64 32 +;; @003e v138 = ushr v136, v137 ; v137 = 32 +;; @003e v139 = iconst.i64 5 +;; @003e v140 = icmp eq v138, v139 ; v139 = 5 +;; @003e brif v140, block11, block12 ;; ;; block11 cold: -;; @003e v140 = iconst.i64 136 -;; @003e v141 = iadd.i64 v27, v140 ; v140 = 136 -;; @003e v142 = load.i64 notrap aligned region2 v141+8 -;; @003e v143 = load.i32 notrap aligned region4 v142 -;; @003e v144 = iconst.i32 0 -;; @003e store notrap aligned region2 v144, v141 ; v144 = 0 +;; @003e v141 = iconst.i64 144 +;; @003e v142 = iadd.i64 v28, v141 ; v141 = 144 +;; @003e v143 = load.i64 notrap aligned region2 v142+8 +;; @003e v144 = load.i32 notrap aligned region4 v143 ;; @003e v145 = iconst.i32 0 -;; @003e store notrap aligned region2 v145, v141+4 ; v145 = 0 -;; @003e v146 = iconst.i64 0 -;; @003e store notrap aligned region2 v146, v141+8 ; v146 = 0 -;; @003e try_call fn2(v0, v143), sig2, block13, [ context v0 ] +;; @003e store notrap aligned region2 v145, v142 ; v145 = 0 +;; @003e v146 = iconst.i32 0 +;; @003e store notrap aligned region2 v146, v142+4 ; v146 = 0 +;; @003e v147 = iconst.i64 0 +;; @003e store notrap aligned region2 v147, v142+8 ; v147 = 0 +;; @003e try_call fn2(v0, v144), sig2, block13, [ context v0 ] ;; ;; block13: ;; @003e trap user12 ;; ;; block12: -;; @003e v147 = iconst.i64 136 -;; @003e v148 = iadd.i64 v27, v147 ; v147 = 136 -;; @003e v149 = load.i64 notrap aligned region2 v148+8 -;; @003e v150 = iconst.i32 0 -;; @003e store notrap aligned region2 v150, v148 ; v150 = 0 +;; @003e v148 = iconst.i64 144 +;; @003e v149 = iadd.i64 v28, v148 ; v148 = 144 +;; @003e v150 = load.i64 notrap aligned region2 v149+8 ;; @003e v151 = iconst.i32 0 -;; @003e store notrap aligned region2 v151, v148+4 ; v151 = 0 -;; @003e v152 = iconst.i64 0 -;; @003e store notrap aligned region2 v152, v148+8 ; v152 = 0 +;; @003e store notrap aligned region2 v151, v149 ; v151 = 0 +;; @003e v152 = iconst.i32 0 +;; @003e store notrap aligned region2 v152, v149+4 ; v152 = 0 +;; @003e v153 = iconst.i64 0 +;; @003e store notrap aligned region2 v153, v149+8 ; v153 = 0 ;; @0041 jump block1 ;; ;; block1: @@ -304,7 +305,7 @@ ;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 ;; gv2 = load.i64 notrap aligned region1 gv1+24 ;; sig0 = (i64 vmctx, i32) -> i64 tail -;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail ;; sig2 = (i64 vmctx) tail ;; fn0 = colocated u805306368:6 sig0 ;; fn1 = colocated u805306368:42 sig1 @@ -317,222 +318,223 @@ ;; @0049 trapz v3, user16 ;; @0049 v4 = iconst.i32 0 ;; @0049 v5 = iconst.i32 0 -;; @0049 v6 = call fn1(v0, v3, v4, v5) ; v4 = 0, v5 = 0 -;; @0049 v7 = load.i64 notrap aligned region2 v6+88 -;; @0049 v8 = uextend.i128 v6 +;; @0049 v6 = iconst.i32 0 +;; @0049 v7 = call fn1(v0, v3, v4, v5, v6) ; v4 = 0, v5 = 0, v6 = 0 +;; @0049 v8 = load.i64 notrap aligned region2 v7+88 ;; @0049 v9 = uextend.i128 v7 -;; @0049 v10 = iconst.i64 64 -;; @0049 v11 = uextend.i128 v10 ; v10 = 64 -;; @0049 v12 = ishl v9, v11 -;; @0049 v13 = bor v12, v8 +;; @0049 v10 = uextend.i128 v8 +;; @0049 v11 = iconst.i64 64 +;; @0049 v12 = uextend.i128 v11 ; v11 = 64 +;; @0049 v13 = ishl v10, v12 +;; @0049 v14 = bor v13, v9 ;; @004b jump block2 ;; ;; block2: -;; @004b v14 = ireduce.i64 v13 -;; @004b v15 = iconst.i64 64 -;; @004b v16 = uextend.i128 v15 ; v15 = 64 -;; @004b v17 = ushr.i128 v13, v16 -;; @004b v18 = ireduce.i64 v17 -;; @004b trapz v14, user16 -;; @004b v19 = load.i64 notrap aligned region2 v14+88 -;; @004b v20 = icmp eq v19, v18 -;; @004b trapz v20, user23 -;; @004b v21 = iconst.i64 1 -;; @004b v22 = iadd v19, v21 ; v21 = 1 -;; @004b store notrap aligned region2 v22, v14+88 -;; @004b v23 = load.i64 notrap aligned region2 v14+80 -;; @004b v24 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @004b v25 = load.i64 notrap aligned region3 v24+88 -;; @004b v26 = load.i64 notrap aligned region3 v24+96 -;; @004b store notrap aligned region2 v25, v23+64 -;; @004b store notrap aligned region2 v26, v23+72 -;; @004b v27 = iconst.i64 0 -;; @004b store notrap aligned region2 v27, v14+80 ; v27 = 0 -;; @004b v28 = iconst.i64 2 -;; @004b v29 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @004b store notrap aligned region3 v28, v29+88 ; v28 = 2 -;; @004b store notrap aligned region3 v14, v29+96 -;; @004b v30 = iconst.i64 0 -;; @004b v31 = iadd v14, v30 ; v30 = 0 -;; @004b v32 = iconst.i32 1 -;; @004b v33 = iconst.i64 32 -;; @004b v34 = iadd v31, v33 ; v33 = 32 -;; @004b store notrap aligned region2 v32, v34 ; v32 = 1 -;; @004b v35 = iconst.i32 2 -;; @004b v36 = iconst.i64 32 -;; @004b v37 = iadd v26, v36 ; v36 = 32 -;; @004b store notrap aligned region2 v35, v37 ; v35 = 2 -;; @004b v38 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @004b v39 = iconst.i64 0 -;; @004b v40 = iadd v26, v39 ; v39 = 0 -;; @004b v41 = load.i64 notrap aligned region4 v38+72 -;; @004b v42 = load.i64 notrap aligned region5 v38+64 -;; @004b v43 = load.i64 notrap aligned region6 v38+80 -;; @004b store notrap aligned region2 v41, v40+8 -;; @004b store notrap aligned region2 v42, v40+16 -;; @004b store notrap aligned region2 v43, v40+24 -;; @004b v44 = load.i64 notrap aligned region1 v38+24 -;; @004b store notrap aligned region2 v44, v40 -;; @004b v45 = iconst.i64 0 -;; @004b v46 = iadd v31, v45 ; v45 = 0 -;; @004b v47 = load.i64 notrap aligned region2 v46 -;; @004b store notrap aligned region1 v47, v38+24 -;; @004b v48 = load.i64 notrap aligned region2 v46+8 -;; @004b store notrap aligned region4 v48, v38+72 -;; @004b v49 = load.i64 notrap aligned region2 v46+16 -;; @004b store notrap aligned region5 v49, v38+64 -;; @004b v50 = load.i64 notrap aligned region2 v46+24 -;; @004b store notrap aligned region6 v50, v38+80 -;; @004b v51 = iconst.i64 40 -;; @004b v52 = iadd v26, v51 ; v51 = 40 -;; @004b v53 = iconst.i32 1 -;; @004b v54 = stack_addr.i64 ss0 -;; @004b store notrap aligned region2 v53, v52+4 ; v53 = 1 -;; @004b store notrap aligned region2 v54, v52+8 -;; @004b v55 = iconst.i64 48 -;; @004b v56 = iadd.i64 v0, v55 ; v55 = 48 -;; @004b v57 = iconst.i32 1 -;; @004b v58 = load.i64 notrap aligned region2 v52+8 -;; @004b store notrap aligned region7 v56, v58 -;; @004b store notrap aligned region2 v57, v52 ; v57 = 1 -;; @004b v59 = iconst.i32 0 -;; @004b store notrap aligned region2 v59, v26+56 ; v59 = 0 -;; @004b v60 = iconst.i64 1 -;; @004b v61 = iconst.i64 32 -;; @004b v62 = ishl v60, v61 ; v60 = 1, v61 = 32 -;; @004b v63 = iconst.i64 96 -;; @004b v64 = iadd v23, v63 ; v63 = 96 -;; @004b v65 = load.i64 notrap aligned region2 v64 -;; @004b v66 = iconst.i64 -24 -;; @004b v67 = iadd v65, v66 ; v66 = -24 -;; @004b v68 = stack_switch v67, v67, v62 -;; @004b v69 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @004b v70 = load.i64 notrap aligned region3 v69+88 -;; @004b v71 = load.i64 notrap aligned region3 v69+96 -;; @004b v72 = load.i64 notrap aligned readonly can_move region0 v0+8 -;; @004b store notrap aligned region3 v25, v72+88 -;; @004b store notrap aligned region3 v26, v72+96 -;; @004b v73 = iconst.i32 1 -;; @004b v74 = iconst.i64 32 -;; @004b v75 = iadd v26, v74 ; v74 = 32 -;; @004b store notrap aligned region2 v73, v75 ; v73 = 1 -;; @004b v76 = iconst.i32 0 -;; @004b store notrap aligned region2 v76, v52 ; v76 = 0 +;; @004b v15 = ireduce.i64 v14 +;; @004b v16 = iconst.i64 64 +;; @004b v17 = uextend.i128 v16 ; v16 = 64 +;; @004b v18 = ushr.i128 v14, v17 +;; @004b v19 = ireduce.i64 v18 +;; @004b trapz v15, user16 +;; @004b v20 = load.i64 notrap aligned region2 v15+88 +;; @004b v21 = icmp eq v20, v19 +;; @004b trapz v21, user23 +;; @004b v22 = iconst.i64 1 +;; @004b v23 = iadd v20, v22 ; v22 = 1 +;; @004b store notrap aligned region2 v23, v15+88 +;; @004b v24 = load.i64 notrap aligned region2 v15+80 +;; @004b v25 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @004b v26 = load.i64 notrap aligned region3 v25+88 +;; @004b v27 = load.i64 notrap aligned region3 v25+96 +;; @004b store notrap aligned region2 v26, v24+64 +;; @004b store notrap aligned region2 v27, v24+72 +;; @004b v28 = iconst.i64 0 +;; @004b store notrap aligned region2 v28, v15+80 ; v28 = 0 +;; @004b v29 = iconst.i64 2 +;; @004b v30 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @004b store notrap aligned region3 v29, v30+88 ; v29 = 2 +;; @004b store notrap aligned region3 v15, v30+96 +;; @004b v31 = iconst.i64 0 +;; @004b v32 = iadd v15, v31 ; v31 = 0 +;; @004b v33 = iconst.i32 1 +;; @004b v34 = iconst.i64 32 +;; @004b v35 = iadd v32, v34 ; v34 = 32 +;; @004b store notrap aligned region2 v33, v35 ; v33 = 1 +;; @004b v36 = iconst.i32 2 +;; @004b v37 = iconst.i64 32 +;; @004b v38 = iadd v27, v37 ; v37 = 32 +;; @004b store notrap aligned region2 v36, v38 ; v36 = 2 +;; @004b v39 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @004b v40 = iconst.i64 0 +;; @004b v41 = iadd v27, v40 ; v40 = 0 +;; @004b v42 = load.i64 notrap aligned region4 v39+72 +;; @004b v43 = load.i64 notrap aligned region5 v39+64 +;; @004b v44 = load.i64 notrap aligned region6 v39+80 +;; @004b store notrap aligned region2 v42, v41+8 +;; @004b store notrap aligned region2 v43, v41+16 +;; @004b store notrap aligned region2 v44, v41+24 +;; @004b v45 = load.i64 notrap aligned region1 v39+24 +;; @004b store notrap aligned region2 v45, v41 +;; @004b v46 = iconst.i64 0 +;; @004b v47 = iadd v32, v46 ; v46 = 0 +;; @004b v48 = load.i64 notrap aligned region2 v47 +;; @004b store notrap aligned region1 v48, v39+24 +;; @004b v49 = load.i64 notrap aligned region2 v47+8 +;; @004b store notrap aligned region4 v49, v39+72 +;; @004b v50 = load.i64 notrap aligned region2 v47+16 +;; @004b store notrap aligned region5 v50, v39+64 +;; @004b v51 = load.i64 notrap aligned region2 v47+24 +;; @004b store notrap aligned region6 v51, v39+80 +;; @004b v52 = iconst.i64 40 +;; @004b v53 = iadd v27, v52 ; v52 = 40 +;; @004b v54 = iconst.i32 1 +;; @004b v55 = stack_addr.i64 ss0 +;; @004b store notrap aligned region2 v54, v53+4 ; v54 = 1 +;; @004b store notrap aligned region2 v55, v53+8 +;; @004b v56 = iconst.i64 48 +;; @004b v57 = iadd.i64 v0, v56 ; v56 = 48 +;; @004b v58 = iconst.i32 1 +;; @004b v59 = load.i64 notrap aligned region2 v53+8 +;; @004b store notrap aligned region7 v57, v59 +;; @004b store notrap aligned region2 v58, v53 ; v58 = 1 +;; @004b v60 = iconst.i32 0 +;; @004b store notrap aligned region2 v60, v27+56 ; v60 = 0 +;; @004b v61 = iconst.i64 1 +;; @004b v62 = iconst.i64 32 +;; @004b v63 = ishl v61, v62 ; v61 = 1, v62 = 32 +;; @004b v64 = iconst.i64 96 +;; @004b v65 = iadd v24, v64 ; v64 = 96 +;; @004b v66 = load.i64 notrap aligned region2 v65 +;; @004b v67 = iconst.i64 -24 +;; @004b v68 = iadd v66, v67 ; v67 = -24 +;; @004b v69 = stack_switch v68, v68, v63 +;; @004b v70 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @004b v71 = load.i64 notrap aligned region3 v70+88 +;; @004b v72 = load.i64 notrap aligned region3 v70+96 +;; @004b v73 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @004b store notrap aligned region3 v26, v73+88 +;; @004b store notrap aligned region3 v27, v73+96 +;; @004b v74 = iconst.i32 1 +;; @004b v75 = iconst.i64 32 +;; @004b v76 = iadd v27, v75 ; v75 = 32 +;; @004b store notrap aligned region2 v74, v76 ; v74 = 1 ;; @004b v77 = iconst.i32 0 -;; @004b store notrap aligned region2 v77, v52+4 ; v77 = 0 -;; @004b v78 = iconst.i64 0 -;; @004b store notrap aligned region2 v78, v52+8 ; v78 = 0 -;; @004b store notrap aligned region2 v27, v26+56 ; v27 = 0 -;; @004b brif v68, block6, block3 +;; @004b store notrap aligned region2 v77, v53 ; v77 = 0 +;; @004b v78 = iconst.i32 0 +;; @004b store notrap aligned region2 v78, v53+4 ; v78 = 0 +;; @004b v79 = iconst.i64 0 +;; @004b store notrap aligned region2 v79, v53+8 ; v79 = 0 +;; @004b store notrap aligned region2 v28, v27+56 ; v28 = 0 +;; @004b brif v69, block6, block3 ;; ;; block6: -;; @004b v79 = iconst.i64 32 -;; @004b v80 = ushr.i64 v68, v79 ; v79 = 32 -;; @004b v81 = iconst.i64 4 -;; @004b v82 = icmp eq v80, v81 ; v81 = 4 -;; @004b brif v82, block5, block4 +;; @004b v80 = iconst.i64 32 +;; @004b v81 = ushr.i64 v69, v80 ; v80 = 32 +;; @004b v82 = iconst.i64 4 +;; @004b v83 = icmp eq v81, v82 ; v82 = 4 +;; @004b brif v83, block5, block4 ;; ;; block5 cold: -;; @004b v83 = iconst.i64 0 -;; @004b v84 = iadd.i64 v71, v83 ; v83 = 0 -;; @004b v85 = iconst.i32 5 -;; @004b v86 = iconst.i64 32 -;; @004b v87 = iadd v84, v86 ; v86 = 32 -;; @004b store notrap aligned region2 v85, v87 ; v85 = 5 -;; @004b v88 = iconst.i64 0 -;; @004b v89 = iadd.i64 v26, v88 ; v88 = 0 -;; @004b v90 = load.i64 notrap aligned region2 v89 -;; @004b store notrap aligned region1 v90, v38+24 -;; @004b v91 = load.i64 notrap aligned region2 v89+8 -;; @004b store notrap aligned region4 v91, v38+72 -;; @004b v92 = load.i64 notrap aligned region2 v89+16 -;; @004b store notrap aligned region5 v92, v38+64 -;; @004b v93 = load.i64 notrap aligned region2 v89+24 -;; @004b store notrap aligned region6 v93, v38+80 -;; @004b v94 = iconst.i64 120 -;; @004b v95 = iadd.i64 v71, v94 ; v94 = 120 -;; @004b v96 = iconst.i32 0 -;; @004b store notrap aligned region2 v96, v95 ; v96 = 0 +;; @004b v84 = iconst.i64 0 +;; @004b v85 = iadd.i64 v72, v84 ; v84 = 0 +;; @004b v86 = iconst.i32 5 +;; @004b v87 = iconst.i64 32 +;; @004b v88 = iadd v85, v87 ; v87 = 32 +;; @004b store notrap aligned region2 v86, v88 ; v86 = 5 +;; @004b v89 = iconst.i64 0 +;; @004b v90 = iadd.i64 v27, v89 ; v89 = 0 +;; @004b v91 = load.i64 notrap aligned region2 v90 +;; @004b store notrap aligned region1 v91, v39+24 +;; @004b v92 = load.i64 notrap aligned region2 v90+8 +;; @004b store notrap aligned region4 v92, v39+72 +;; @004b v93 = load.i64 notrap aligned region2 v90+16 +;; @004b store notrap aligned region5 v93, v39+64 +;; @004b v94 = load.i64 notrap aligned region2 v90+24 +;; @004b store notrap aligned region6 v94, v39+80 +;; @004b v95 = iconst.i64 120 +;; @004b v96 = iadd.i64 v72, v95 ; v95 = 120 ;; @004b v97 = iconst.i32 0 -;; @004b store notrap aligned region2 v97, v95+4 ; v97 = 0 -;; @004b v98 = iconst.i64 0 -;; @004b store notrap aligned region2 v98, v95+8 ; v98 = 0 -;; @004b v99 = iconst.i64 136 -;; @004b v100 = iadd.i64 v71, v99 ; v99 = 136 -;; @004b v101 = iconst.i32 0 -;; @004b store notrap aligned region2 v101, v100 ; v101 = 0 +;; @004b store notrap aligned region2 v97, v96 ; v97 = 0 +;; @004b v98 = iconst.i32 0 +;; @004b store notrap aligned region2 v98, v96+4 ; v98 = 0 +;; @004b v99 = iconst.i64 0 +;; @004b store notrap aligned region2 v99, v96+8 ; v99 = 0 +;; @004b v100 = iconst.i64 144 +;; @004b v101 = iadd.i64 v72, v100 ; v100 = 144 ;; @004b v102 = iconst.i32 0 -;; @004b store notrap aligned region2 v102, v100+4 ; v102 = 0 -;; @004b v103 = iconst.i64 0 -;; @004b store notrap aligned region2 v103, v100+8 ; v103 = 0 +;; @004b store notrap aligned region2 v102, v101 ; v102 = 0 +;; @004b v103 = iconst.i32 0 +;; @004b store notrap aligned region2 v103, v101+4 ; v103 = 0 +;; @004b v104 = iconst.i64 0 +;; @004b store notrap aligned region2 v104, v101+8 ; v104 = 0 ;; @004b try_call fn2(v0), sig2, block8, [ context v0 ] ;; ;; block8: ;; @004b trap user12 ;; ;; block4: -;; @004b v104 = iconst.i64 0 -;; @004b v105 = iadd.i64 v71, v104 ; v104 = 0 -;; @004b v106 = iconst.i64 0 -;; @004b v107 = iadd v105, v106 ; v106 = 0 -;; @004b v108 = load.i64 notrap aligned region4 v38+72 -;; @004b v109 = load.i64 notrap aligned region5 v38+64 -;; @004b v110 = load.i64 notrap aligned region6 v38+80 -;; @004b store notrap aligned region2 v108, v107+8 -;; @004b store notrap aligned region2 v109, v107+16 -;; @004b store notrap aligned region2 v110, v107+24 -;; @004b v111 = iconst.i64 0 -;; @004b v112 = iadd.i64 v26, v111 ; v111 = 0 -;; @004b v113 = load.i64 notrap aligned region2 v112 -;; @004b store notrap aligned region1 v113, v38+24 -;; @004b v114 = load.i64 notrap aligned region2 v112+8 -;; @004b store notrap aligned region4 v114, v38+72 -;; @004b v115 = load.i64 notrap aligned region2 v112+16 -;; @004b store notrap aligned region5 v115, v38+64 -;; @004b v116 = load.i64 notrap aligned region2 v112+24 -;; @004b store notrap aligned region6 v116, v38+80 -;; @004b v117 = ireduce.i32 v68 -;; @004b v118 = load.i64 notrap aligned region2 v71+88 -;; @004b v119 = uextend.i128 v71 -;; @004b v120 = uextend.i128 v118 -;; @004b v121 = iconst.i64 64 -;; @004b v122 = uextend.i128 v121 ; v121 = 64 -;; @004b v123 = ishl v120, v122 -;; @004b v124 = bor v123, v119 +;; @004b v105 = iconst.i64 0 +;; @004b v106 = iadd.i64 v72, v105 ; v105 = 0 +;; @004b v107 = iconst.i64 0 +;; @004b v108 = iadd v106, v107 ; v107 = 0 +;; @004b v109 = load.i64 notrap aligned region4 v39+72 +;; @004b v110 = load.i64 notrap aligned region5 v39+64 +;; @004b v111 = load.i64 notrap aligned region6 v39+80 +;; @004b store notrap aligned region2 v109, v108+8 +;; @004b store notrap aligned region2 v110, v108+16 +;; @004b store notrap aligned region2 v111, v108+24 +;; @004b v112 = iconst.i64 0 +;; @004b v113 = iadd.i64 v27, v112 ; v112 = 0 +;; @004b v114 = load.i64 notrap aligned region2 v113 +;; @004b store notrap aligned region1 v114, v39+24 +;; @004b v115 = load.i64 notrap aligned region2 v113+8 +;; @004b store notrap aligned region4 v115, v39+72 +;; @004b v116 = load.i64 notrap aligned region2 v113+16 +;; @004b store notrap aligned region5 v116, v39+64 +;; @004b v117 = load.i64 notrap aligned region2 v113+24 +;; @004b store notrap aligned region6 v117, v39+80 +;; @004b v118 = ireduce.i32 v69 +;; @004b v119 = load.i64 notrap aligned region2 v72+88 +;; @004b v120 = uextend.i128 v72 +;; @004b v121 = uextend.i128 v119 +;; @004b v122 = iconst.i64 64 +;; @004b v123 = uextend.i128 v122 ; v122 = 64 +;; @004b v124 = ishl v121, v123 +;; @004b v125 = bor v124, v120 ;; @004b jump block7 ;; ;; block9 cold: ;; @004b trap user12 ;; ;; block7: -;; @004b br_table v117, block9, [] +;; @004b br_table v118, block9, [] ;; ;; block3: -;; @004b v125 = iconst.i64 0 -;; @004b v126 = iadd.i64 v26, v125 ; v125 = 0 -;; @004b v127 = load.i64 notrap aligned region2 v126 -;; @004b store notrap aligned region1 v127, v38+24 -;; @004b v128 = load.i64 notrap aligned region2 v126+8 -;; @004b store notrap aligned region4 v128, v38+72 -;; @004b v129 = load.i64 notrap aligned region2 v126+16 -;; @004b store notrap aligned region5 v129, v38+64 -;; @004b v130 = load.i64 notrap aligned region2 v126+24 -;; @004b store notrap aligned region6 v130, v38+80 -;; @004b v131 = iconst.i64 0 -;; @004b v132 = iadd.i64 v71, v131 ; v131 = 0 -;; @004b v133 = iconst.i32 4 -;; @004b v134 = iconst.i64 32 -;; @004b v135 = iadd v132, v134 ; v134 = 32 -;; @004b store notrap aligned region2 v133, v135 ; v133 = 4 -;; @004b v136 = iconst.i64 120 -;; @004b v137 = iadd.i64 v71, v136 ; v136 = 120 -;; @004b v138 = load.i64 notrap aligned region2 v137+8 -;; @004b v139 = iconst.i32 0 -;; @004b store notrap aligned region2 v139, v137 ; v139 = 0 +;; @004b v126 = iconst.i64 0 +;; @004b v127 = iadd.i64 v27, v126 ; v126 = 0 +;; @004b v128 = load.i64 notrap aligned region2 v127 +;; @004b store notrap aligned region1 v128, v39+24 +;; @004b v129 = load.i64 notrap aligned region2 v127+8 +;; @004b store notrap aligned region4 v129, v39+72 +;; @004b v130 = load.i64 notrap aligned region2 v127+16 +;; @004b store notrap aligned region5 v130, v39+64 +;; @004b v131 = load.i64 notrap aligned region2 v127+24 +;; @004b store notrap aligned region6 v131, v39+80 +;; @004b v132 = iconst.i64 0 +;; @004b v133 = iadd.i64 v72, v132 ; v132 = 0 +;; @004b v134 = iconst.i32 4 +;; @004b v135 = iconst.i64 32 +;; @004b v136 = iadd v133, v135 ; v135 = 32 +;; @004b store notrap aligned region2 v134, v136 ; v134 = 4 +;; @004b v137 = iconst.i64 120 +;; @004b v138 = iadd.i64 v72, v137 ; v137 = 120 +;; @004b v139 = load.i64 notrap aligned region2 v138+8 ;; @004b v140 = iconst.i32 0 -;; @004b store notrap aligned region2 v140, v137+4 ; v140 = 0 -;; @004b v141 = iconst.i64 0 -;; @004b store notrap aligned region2 v141, v137+8 ; v141 = 0 +;; @004b store notrap aligned region2 v140, v138 ; v140 = 0 +;; @004b v141 = iconst.i32 0 +;; @004b store notrap aligned region2 v141, v138+4 ; v141 = 0 +;; @004b v142 = iconst.i64 0 +;; @004b store notrap aligned region2 v142, v138+8 ; v142 = 0 ;; @0050 jump block1 ;; ;; block1: diff --git a/tests/misc_testsuite/stack-switching-and-gc.wast b/tests/misc_testsuite/stack-switching-and-gc.wast deleted file mode 100644 index 229f2fe16dd1..000000000000 --- a/tests/misc_testsuite/stack-switching-and-gc.wast +++ /dev/null @@ -1,13 +0,0 @@ -;;! stack_switching = true -;;! gc = true - -(assert_invalid - (module - (type $f (func)) - (type $c (cont $f)) - (type $s (struct (field (ref null $c)))) - (func (export "run") - (drop (struct.new_default $s)) - ) - ) - "Stack switching feature not compatible with GC") diff --git a/tests/misc_testsuite/stack-switching/gc.wast b/tests/misc_testsuite/stack-switching/gc.wast new file mode 100644 index 000000000000..5cbe6af86806 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/gc.wast @@ -0,0 +1,363 @@ +;;! stack_switching = true +;;! gc = true +;;! function_references = true +;;! bulk_memory = true + +;; A GC reference in a parent frame must remain live while a child +;; continuation is running. +(module + (type $box (struct (field i32))) + (type $f (func)) + (type $c (cont $f)) + (tag $suspend) + + (import "wasmtime" "gc" (func $gc)) + + (func $child + (call $gc) + (suspend $suspend)) + (elem declare func $child) + + (func (export "parent-frame") (result i32) + (local $box (ref null $box)) + (local.set $box (struct.new $box (i32.const 42))) + + (drop + (block $on-suspend (result (ref $c)) + (resume $c + (on $suspend $on-suspend) + (cont.new $c (ref.func $child))) + (unreachable))) + + (struct.get $box 0 (local.get $box))) +) + +(assert_return (invoke "parent-frame") (i32.const 42)) + +;; Stack-map tracing follows the full parent chain, and execution can +;; subsequently re-enter every parent frame. +(module + (type $box (struct (field i32))) + (type $f (func (result i32))) + (type $c (cont $f)) + (tag $suspend) + + (import "wasmtime" "gc" (func $gc)) + + (func $child (result i32) + (call $gc) + (suspend $suspend) + (i32.const 1)) + (elem declare func $child) + + (func $middle (result i32) + (local $box (ref null $box)) + (local.set $box (struct.new $box (i32.const 20))) + (i32.add + (resume $c (cont.new $c (ref.func $child))) + (struct.get $box 0 (local.get $box)))) + (elem declare func $middle) + + (func (export "parent-chain") (result i32) + (local $box (ref null $box)) + (local $suspended (ref null $c)) + (local.set $box (struct.new $box (i32.const 100))) + + (local.set $suspended + (block $on-suspend (result (ref $c)) + (resume $c + (on $suspend $on-suspend) + (cont.new $c (ref.func $middle))) + (unreachable))) + + (i32.add + (resume $c (local.get $suspended)) + (struct.get $box 0 (local.get $box)))) +) + +(assert_return (invoke "parent-chain") (i32.const 121)) + +;; A bound GC reference is retained only by a fresh continuation's +;; argument buffer when collection runs before the continuation is +;; first resumed. +(module + (type $box (struct (field i32))) + (type $with-arg-f (func (param (ref $box)) (result i32))) + (type $without-arg-f (func (result i32))) + (type $with-arg-c (cont $with-arg-f)) + (type $without-arg-c (cont $without-arg-f)) + + (import "wasmtime" "gc" (func $gc)) + + (func $read-box (param $box (ref $box)) (result i32) + (struct.get $box 0 (local.get $box))) + (elem declare func $read-box) + + (func (export "bound-argument") (result i32) + (local $continuation (ref null $without-arg-c)) + + (local.set $continuation + (cont.bind $with-arg-c $without-arg-c + (struct.new $box (i32.const 73)) + (cont.new $with-arg-c (ref.func $read-box)))) + + (call $gc) + (resume $without-arg-c + (local.get $continuation))) +) + +(assert_return (invoke "bound-argument") (i32.const 73)) + +;; The same root metadata is used for the `values` buffer after a +;; continuation has suspended. +(module + (type $box (struct (field i32))) + (type $initial-f (func (result i32))) + (type $suspended-f (func (param (ref $box)) (result i32))) + (type $initial-c (cont $initial-f)) + (type $suspended-c (cont $suspended-f)) + (tag $yield (result (ref $box))) + + (import "wasmtime" "gc" (func $gc)) + + (func $suspend-then-read (result i32) + (struct.get $box 0 (suspend $yield))) + (elem declare func $suspend-then-read) + + (func (export "bound-suspended-argument") (result i32) + (local $suspended (ref null $suspended-c)) + (local $ready (ref null $initial-c)) + + (local.set $suspended + (block $on-yield (result (ref $suspended-c)) + (resume $initial-c + (on $yield $on-yield) + (cont.new $initial-c (ref.func $suspend-then-read))) + (unreachable))) + + (local.set $ready + (cont.bind $suspended-c $initial-c + (struct.new $box (i32.const 91)) + (local.get $suspended))) + + (call $gc) + (resume $initial-c (ref.as_non_null (local.get $ready)))) +) + +(assert_return (invoke "bound-suspended-argument") (i32.const 91)) + +;; Struct and array fields store continuation references through a +;; store-local side table. Collection may run while the aggregate is +;; the only place that retains the continuation object. +(module + (type $f (func (result i32))) + (type $c (cont $f)) + (type $holder (struct (field (mut (ref null $c))))) + (type $array (array (mut (ref null $c)))) + + (import "wasmtime" "gc" (func $gc)) + + (func $forty-two (result i32) + (i32.const 42)) + (elem declare func $forty-two) + + (func (export "struct-initialize") (result i32) + (local $holder (ref $holder)) + (local.set $holder + (struct.new $holder + (cont.new $c (ref.func $forty-two)))) + (call $gc) + (resume $c + (struct.get $holder 0 (local.get $holder)))) + + (func (export "struct-update") (result i32) + (local $holder (ref $holder)) + (local.set $holder (struct.new_default $holder)) + (struct.set $holder 0 + (local.get $holder) + (cont.new $c (ref.func $forty-two))) + (call $gc) + (resume $c + (struct.get $holder 0 (local.get $holder)))) + + (func (export "array-initialize") (result i32) + (local $array (ref $array)) + (local.set $array + (array.new_fixed $array 1 + (cont.new $c (ref.func $forty-two)))) + (call $gc) + (resume $c + (array.get $array (local.get $array) (i32.const 0)))) + + (func (export "array-update") (result i32) + (local $array (ref $array)) + (local.set $array (array.new_default $array (i32.const 1))) + (array.set $array + (local.get $array) + (i32.const 0) + (cont.new $c (ref.func $forty-two))) + (call $gc) + (resume $c + (array.get $array (local.get $array) (i32.const 0)))) + + (func (export "array-fill-and-copy") (result i32) + (local $source (ref $array)) + (local $destination (ref $array)) + (local.set $source (array.new_default $array (i32.const 1))) + (local.set $destination (array.new_default $array (i32.const 1))) + (array.fill $array + (local.get $source) + (i32.const 0) + (cont.new $c (ref.func $forty-two)) + (i32.const 1)) + (array.copy $array $array + (local.get $destination) + (i32.const 0) + (local.get $source) + (i32.const 0) + (i32.const 1)) + (call $gc) + (resume $c + (array.get $array (local.get $destination) (i32.const 0)))) + + (func (export "null-defaults") (result i32) + (local $holder (ref $holder)) + (local $array (ref $array)) + (local.set $holder (struct.new_default $holder)) + (local.set $array (array.new_default $array (i32.const 1))) + (i32.add + (ref.is_null (struct.get $holder 0 (local.get $holder))) + (ref.is_null + (array.get $array (local.get $array) (i32.const 0))))) + + ;; Loading an alias from a GC object must reproduce the interned + ;; revision witness. Consuming one alias therefore invalidates all + ;; other loads of the same field. + (func (export "consume-struct-field-twice") + (local $holder (ref $holder)) + (local.set $holder + (struct.new $holder + (cont.new $c (ref.func $forty-two)))) + (drop + (resume $c + (struct.get $holder 0 (local.get $holder)))) + (drop + (resume $c + (struct.get $holder 0 (local.get $holder))))) +) + +(assert_return (invoke "struct-initialize") (i32.const 42)) +(assert_return (invoke "struct-update") (i32.const 42)) +(assert_return (invoke "array-initialize") (i32.const 42)) +(assert_return (invoke "array-update") (i32.const 42)) +(assert_return (invoke "array-fill-and-copy") (i32.const 42)) +(assert_return (invoke "null-defaults") (i32.const 2)) +(assert_trap + (invoke "consume-struct-field-twice") + "continuation already consumed") + +;; A GC reference passed to a child continuation must remain live when +;; that child is unwound by `resume_throw`. The parent retains another +;; reference, catches the injected exception, and can still use the +;; object after a second collection. +(module + (type $box (struct (field i32))) + (type $child-f (func (param (ref $box)) (result i32))) + (type $parent-f (func (result i32))) + (type $child-c (cont $child-f)) + (type $parent-c (cont $parent-f)) + (tag $suspend) + (tag $exception) + + (import "wasmtime" "gc" (func $gc)) + + (func $child (param $box (ref $box)) (result i32) + ;; At this collection, both the child and its parent have the box + ;; in live stack slots. Keep the parameter live across the + ;; suspension point. + (call $gc) + (suspend $suspend) + (struct.get $box 0 (local.get $box))) + (elem declare func $child) + + (func $parent (result i32) + (local $box (ref $box)) + (local.set $box (struct.new $box (i32.const 101))) + + (block $caught + (try_table (catch $exception $caught) + (drop + (resume $child-c + (local.get $box) + (cont.new $child-c (ref.func $child))))) + (unreachable)) + + ;; The child continuation is terminal, so only the parent's stack + ;; slot retains the box at this collection. + (call $gc) + (struct.get $box 0 (local.get $box))) + (elem declare func $parent) + + (func (export "gc-ref-survives-resume-throw") (result i32) + (local $suspended (ref null $parent-c)) + + (local.set $suspended + (block $on-suspend (result (ref $parent-c)) + (resume $parent-c + (on $suspend $on-suspend) + (cont.new $parent-c (ref.func $parent))) + (unreachable))) + + (resume_throw $parent-c $exception + (local.get $suspended))) +) + +(assert_return + (invoke "gc-ref-survives-resume-throw") + (i32.const 101)) + +;; A GC reference carried by an exception injected with `resume_throw` +;; must become a live stack root when the exception is caught inside +;; the resumed continuation. +(module + (type $box (struct (field i32))) + (type $f (func (result i32))) + (type $c (cont $f)) + (tag $suspend) + (tag $exception (param (ref $box))) + + (import "wasmtime" "gc" (func $gc)) + + (func $catch-payload (result i32) + (local $box (ref null $box)) + + (local.set $box + (block $caught (result (ref $box)) + (try_table (result (ref $box)) (catch $exception $caught) + (suspend $suspend) + (unreachable)))) + + ;; The caught local is now the only reference to the exception + ;; payload. + (call $gc) + (struct.get $box 0 (ref.as_non_null (local.get $box)))) + (elem declare func $catch-payload) + + (func (export "gc-ref-in-resume-throw-payload") (result i32) + (local $suspended (ref null $c)) + + (local.set $suspended + (block $on-suspend (result (ref $c)) + (resume $c + (on $suspend $on-suspend) + (cont.new $c (ref.func $catch-payload))) + (unreachable))) + + (resume_throw $c $exception + (struct.new $box (i32.const 202)) + (local.get $suspended))) +) + +(assert_return + (invoke "gc-ref-in-resume-throw-payload") + (i32.const 202)) diff --git a/tests/misc_testsuite/stack-switching/issue13021.wast b/tests/misc_testsuite/stack-switching/issue13021.wast new file mode 100644 index 000000000000..949362e0190a --- /dev/null +++ b/tests/misc_testsuite/stack-switching/issue13021.wast @@ -0,0 +1,18 @@ +;;! gc = true +;;! stack_switching = true + +;; Regression test for issue https://github.com/bytecodealliance/wasmtime/issues/13021 +(module + (type $ft (func)) + (type $ct (cont $ft)) + (type $arr (array (mut (ref null $ct)))) + (func (export "boom") + (local $a (ref $arr)) + (local.set $a (array.new_default $arr (i32.const 2))) + (array.copy $arr $arr + (local.get $a) (i32.const 0) + (local.get $a) (i32.const 0) + (i32.const 1)) + ) +) +(assert_return (invoke "boom")) \ No newline at end of file diff --git a/tests/misc_testsuite/stack-switching/issue13022.wast b/tests/misc_testsuite/stack-switching/issue13022.wast new file mode 100644 index 000000000000..1ccf5a7c6991 --- /dev/null +++ b/tests/misc_testsuite/stack-switching/issue13022.wast @@ -0,0 +1,18 @@ +;;! gc = true +;;! stack_switching = true + +;; Regression test for https://github.com/bytecodealliance/wasmtime/issues/13022 +(module + (type $ft (func)) + (type $ct (cont $ft)) + (type $arr (array (mut (ref null $ct)))) + (func (export "boom") + (local $a (ref $arr)) + (local.set $a (array.new_default $arr (i32.const 2))) + (array.copy $arr $arr + (local.get $a) (i32.const 0) + (local.get $a) (i32.const 0) + (i32.const 1)) + ) +) +(assert_return (invoke "boom")) \ No newline at end of file