| From: rive | |
| Subject: [PATCH 0040] parallel call members over a shared memory | |
| A parallel call runs one function on extra instances of the main module | |
| that share its memory. Members skip the start function, since the host | |
| copies the main instance's globals into them before each run. The fast | |
| interpreter's atomics no longer take the process wide shared memory mutex, | |
| and neither do pointer checks on a shared memory, which never moves. | |
| diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c | |
| index 3d552cf..00ada3f 100644 | |
| --- a/core/iwasm/aot/aot_runtime.c | |
| +++ b/core/iwasm/aot/aot_runtime.c | |
| @@ -1743,6 +1743,11 @@ execute_post_instantiate_functions(AOTModuleInstance *module_inst, | |
| } | |
| #endif | |
| + /* rive: a parallel call member has no exec env to run them on, and the | |
| + host copies in the globals they would have set */ | |
| + if (is_sub_inst && !exec_env_main) | |
| + return true; | |
| + | |
| if (!module->start_function && !initialize_func && !post_inst_func | |
| && !call_ctors_func) { | |
| /* No post instantiation functions to call */ | |
| diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c | |
| index ba2acc3..2f72446 100644 | |
| --- a/core/iwasm/common/wasm_memory.c | |
| +++ b/core/iwasm/common/wasm_memory.c | |
| @@ -999,6 +999,17 @@ wasm_runtime_get_mem_alloc_info(mem_alloc_info_t *mem_alloc_info) | |
| return false; | |
| } | |
| +/* rive: a shared memory never moves and only grows, so a check racing a | |
| + growth at worst refuses bytes just added; with atomic 64 bit sizes the | |
| + process wide mutex was only overhead on every host call with a pointer */ | |
| +#if WASM_ENABLE_SHARED_MEMORY != 0 && BH_ATOMIC_64_IS_ATOMIC != 0 | |
| +#define VALIDATE_LOCK(memory) (void)0 | |
| +#define VALIDATE_UNLOCK(memory) (void)0 | |
| +#else | |
| +#define VALIDATE_LOCK(memory) SHARED_MEMORY_LOCK(memory) | |
| +#define VALIDATE_UNLOCK(memory) SHARED_MEMORY_UNLOCK(memory) | |
| +#endif | |
| + | |
| bool | |
| wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, | |
| uint64 app_offset, uint64 size) | |
| @@ -1036,14 +1047,14 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, | |
| goto fail; | |
| } | |
| - SHARED_MEMORY_LOCK(memory_inst); | |
| + VALIDATE_LOCK(memory_inst); | |
| if (app_offset + size <= memory_inst->memory_data_size) { | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| return true; | |
| } | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| fail: | |
| wasm_set_exception(module_inst, "out of bounds memory access"); | |
| @@ -1154,15 +1165,15 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, | |
| } | |
| #endif | |
| - SHARED_MEMORY_LOCK(memory_inst); | |
| + VALIDATE_LOCK(memory_inst); | |
| if (memory_inst->memory_data <= addr | |
| && addr + size <= memory_inst->memory_data_end) { | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| return true; | |
| } | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| fail: | |
| wasm_set_exception(module_inst, "out of bounds memory access"); | |
| @@ -1196,22 +1207,22 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, | |
| } | |
| #endif | |
| - SHARED_MEMORY_LOCK(memory_inst); | |
| + VALIDATE_LOCK(memory_inst); | |
| addr = memory_inst->memory_data + (uintptr_t)app_offset; | |
| if (bounds_checks) { | |
| if (memory_inst->memory_data <= addr | |
| && addr < memory_inst->memory_data_end) { | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| return addr; | |
| } | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| return NULL; | |
| } | |
| /* If bounds checks is disabled, return the address directly */ | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| return addr; | |
| } | |
| @@ -1244,24 +1255,24 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, | |
| } | |
| #endif | |
| - SHARED_MEMORY_LOCK(memory_inst); | |
| + VALIDATE_LOCK(memory_inst); | |
| if (bounds_checks) { | |
| if (memory_inst->memory_data <= addr | |
| && addr < memory_inst->memory_data_end) { | |
| ret = (uint64)(addr - memory_inst->memory_data); | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| return ret; | |
| } | |
| } | |
| /* If bounds checks is disabled, return the offset directly */ | |
| else if (addr != NULL) { | |
| ret = (uint64)(addr - memory_inst->memory_data); | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| return ret; | |
| } | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| return 0; | |
| } | |
| @@ -1282,7 +1293,7 @@ wasm_runtime_get_app_addr_range(WASMModuleInstanceCommon *module_inst_comm, | |
| return false; | |
| } | |
| - SHARED_MEMORY_LOCK(memory_inst); | |
| + VALIDATE_LOCK(memory_inst); | |
| memory_data_size = memory_inst->memory_data_size; | |
| @@ -1291,11 +1302,11 @@ wasm_runtime_get_app_addr_range(WASMModuleInstanceCommon *module_inst_comm, | |
| *p_app_start_offset = 0; | |
| if (p_app_end_offset) | |
| *p_app_end_offset = memory_data_size; | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| return true; | |
| } | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| return false; | |
| } | |
| @@ -1317,7 +1328,7 @@ wasm_runtime_get_native_addr_range(WASMModuleInstanceCommon *module_inst_comm, | |
| return false; | |
| } | |
| - SHARED_MEMORY_LOCK(memory_inst); | |
| + VALIDATE_LOCK(memory_inst); | |
| if (memory_inst->memory_data <= addr | |
| && addr < memory_inst->memory_data_end) { | |
| @@ -1325,11 +1336,11 @@ wasm_runtime_get_native_addr_range(WASMModuleInstanceCommon *module_inst_comm, | |
| *p_native_start_addr = memory_inst->memory_data; | |
| if (p_native_end_addr) | |
| *p_native_end_addr = memory_inst->memory_data_end; | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| return true; | |
| } | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| return false; | |
| } | |
| @@ -1393,7 +1404,7 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, | |
| /* rive patch: validate strictly even in hw-bounds builds; a permissive | |
| pass here turns a bad app pointer into a guard-page longjmp from | |
| inside a host native's frame. */ | |
| - SHARED_MEMORY_LOCK(memory_inst); | |
| + VALIDATE_LOCK(memory_inst); | |
| if (app_buf_addr >= memory_inst->memory_data_size) { | |
| goto fail; | |
| @@ -1416,14 +1427,14 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, | |
| goto fail; | |
| } | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| success: | |
| *p_native_addr = (void *)native_addr; | |
| return true; | |
| fail: | |
| - SHARED_MEMORY_UNLOCK(memory_inst); | |
| + VALIDATE_UNLOCK(memory_inst); | |
| wasm_set_exception(module_inst, "out of bounds memory access"); | |
| return false; | |
| } | |
| diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h | |
| index 2bd0951..0c71e16 100644 | |
| --- a/core/iwasm/common/wasm_memory.h | |
| +++ b/core/iwasm/common/wasm_memory.h | |
| @@ -25,7 +25,9 @@ static inline uint64 | |
| GET_LINEAR_MEMORY_SIZE(const WASMMemoryInstance *memory) | |
| { | |
| SHARED_MEMORY_LOCK(memory); | |
| - uint64 memory_data_size = BH_ATOMIC_64_LOAD(memory->memory_data_size); | |
| + /* rive: a real load per check, as below for builds without shared | |
| + memory */ | |
| + uint64 memory_data_size = *(volatile uint64 *)&memory->memory_data_size; | |
| SHARED_MEMORY_UNLOCK(memory); | |
| return memory_data_size; | |
| } | |
| diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c | |
| index 434271d..0940558 100644 | |
| --- a/core/iwasm/common/wasm_runtime_common.c | |
| +++ b/core/iwasm/common/wasm_runtime_common.c | |
| @@ -3285,6 +3285,47 @@ wasm_runtime_rive_clear_interrupt(WASMExecEnv *exec_env) | |
| ~WASM_SUSPEND_FLAG_TERMINATE); | |
| } | |
| +bool | |
| +wasm_runtime_rive_interrupted(WASMExecEnv *exec_env) | |
| +{ | |
| + return (WASM_SUSPEND_FLAGS_GET(exec_env->suspend_flags) | |
| + & WASM_SUSPEND_FLAG_TERMINATE) | |
| + != 0; | |
| +} | |
| + | |
| +WASMModuleInstanceCommon * | |
| +wasm_runtime_instantiate_parallel_member(WASMModuleInstanceCommon *parent, | |
| + uint32 stack_size, char *error_buf, | |
| + uint32 error_buf_size) | |
| +{ | |
| + WASMModuleInstance *parent_inst = (WASMModuleInstance *)parent; | |
| + | |
| +#if WASM_ENABLE_SHARED_MEMORY != 0 | |
| + if (parent_inst->memory_count == 1 | |
| + && shared_memory_is_shared(parent_inst->memories[0])) | |
| + return wasm_runtime_instantiate_internal( | |
| + (WASMModuleCommon *)parent_inst->module, parent, NULL, stack_size, | |
| + 0, 0, error_buf, error_buf_size); | |
| +#endif | |
| + (void)parent_inst; | |
| + (void)stack_size; | |
| + set_error_buf(error_buf, error_buf_size, | |
| + "parallel call members need one shared memory"); | |
| + return NULL; | |
| +} | |
| + | |
| +void | |
| +wasm_runtime_copy_globals(WASMModuleInstanceCommon *dst, | |
| + WASMModuleInstanceCommon *src) | |
| +{ | |
| + WASMModuleInstance *dst_inst = (WASMModuleInstance *)dst; | |
| + WASMModuleInstance *src_inst = (WASMModuleInstance *)src; | |
| + | |
| + bh_assert(dst_inst->module == src_inst->module); | |
| + bh_memcpy_s(dst_inst->global_data, dst_inst->global_data_size, | |
| + src_inst->global_data, src_inst->global_data_size); | |
| +} | |
| + | |
| static wasm_rive_start_hook_t rive_start_hook = NULL; | |
| static void *rive_start_hook_data = NULL; | |
| diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h | |
| index 3c08264..6007561 100644 | |
| --- a/core/iwasm/include/wasm_export.h | |
| +++ b/core/iwasm/include/wasm_export.h | |
| @@ -1341,6 +1341,30 @@ wasm_runtime_rive_interrupt(wasm_exec_env_t exec_env); | |
| WASM_RUNTIME_API_EXTERN void | |
| wasm_runtime_rive_clear_interrupt(wasm_exec_env_t exec_env); | |
| +/** | |
| + * rive: whether exec_env has an interrupt pending. | |
| + */ | |
| +WASM_RUNTIME_API_EXTERN bool | |
| +wasm_runtime_rive_interrupted(wasm_exec_env_t exec_env); | |
| + | |
| +/** | |
| + * rive: instantiate parent's module again over parent's shared memory 0, as | |
| + * a parallel call member with its own globals, tables and exception state. | |
| + * Data segments and the start function are skipped, so the host copies the | |
| + * parent's globals in before each run. Members must be deinstantiated | |
| + * before parent. | |
| + */ | |
| +WASM_RUNTIME_API_EXTERN wasm_module_inst_t | |
| +wasm_runtime_instantiate_parallel_member(wasm_module_inst_t parent, | |
| + uint32_t stack_size, char *error_buf, | |
| + uint32_t error_buf_size); | |
| + | |
| +/** | |
| + * rive: copy src's global values into dst, an instance of the same module. | |
| + */ | |
| +WASM_RUNTIME_API_EXTERN void | |
| +wasm_runtime_copy_globals(wasm_module_inst_t dst, wasm_module_inst_t src); | |
| + | |
| /** | |
| * rive: called on the instantiating thread with the exec env a module's | |
| * start function runs on, before it runs (begin true) and after (begin | |
| diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c | |
| index 2158b16..e195899 100644 | |
| --- a/core/iwasm/interpreter/wasm_interp_fast.c | |
| +++ b/core/iwasm/interpreter/wasm_interp_fast.c | |
| @@ -745,6 +745,29 @@ wasm_interp_get_frame_ref(WASMInterpFrame *frame) | |
| } while (0) | |
| #endif | |
| +/* rive: lock free atomics; the global shared memory mutex serialized every | |
| + parallel call member's atomics */ | |
| +#define ATOMIC_LOAD(T) __atomic_load_n((T *)maddr, __ATOMIC_SEQ_CST) | |
| +#define ATOMIC_STORE(T, value) \ | |
| + __atomic_store_n((T *)maddr, (T)(value), __ATOMIC_SEQ_CST) | |
| +#define ATOMIC_CMPXCHG(T, expected, value, result) \ | |
| + do { \ | |
| + T seen_ = (T)(expected); \ | |
| + __atomic_compare_exchange_n((T *)maddr, &seen_, (T)(value), false, \ | |
| + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); \ | |
| + (result) = seen_; \ | |
| + } while (0) | |
| +#define ATOMIC_RMW(T, op, result) \ | |
| + do { \ | |
| + T old_ = __atomic_load_n((T *)maddr, __ATOMIC_SEQ_CST); \ | |
| + while (!__atomic_compare_exchange_n((T *)maddr, &old_, \ | |
| + (T)(old_ op sval), true, \ | |
| + __ATOMIC_SEQ_CST, \ | |
| + __ATOMIC_SEQ_CST)) { \ | |
| + } \ | |
| + (result) = old_; \ | |
| + } while (0) | |
| + | |
| #define DEF_ATOMIC_RMW_OPCODE(OP_NAME, op) \ | |
| case WASM_OP_ATOMIC_RMW_I32_##OP_NAME: \ | |
| case WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U: \ | |
| @@ -759,28 +782,19 @@ wasm_interp_get_frame_ref(WASMInterpFrame *frame) | |
| CHECK_MEMORY_OVERFLOW(1); \ | |
| CHECK_ATOMIC_MEMORY_ACCESS(1); \ | |
| \ | |
| - shared_memory_lock(memory); \ | |
| - readv = (uint32)(*(uint8 *)maddr); \ | |
| - *(uint8 *)maddr = (uint8)(readv op sval); \ | |
| - shared_memory_unlock(memory); \ | |
| + ATOMIC_RMW(uint8, op, readv); \ | |
| } \ | |
| else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \ | |
| CHECK_MEMORY_OVERFLOW(2); \ | |
| CHECK_ATOMIC_MEMORY_ACCESS(2); \ | |
| \ | |
| - shared_memory_lock(memory); \ | |
| - readv = (uint32)LOAD_U16(maddr); \ | |
| - STORE_U16(maddr, (uint16)(readv op sval)); \ | |
| - shared_memory_unlock(memory); \ | |
| + ATOMIC_RMW(uint16, op, readv); \ | |
| } \ | |
| else { \ | |
| CHECK_MEMORY_OVERFLOW(4); \ | |
| CHECK_ATOMIC_MEMORY_ACCESS(4); \ | |
| \ | |
| - shared_memory_lock(memory); \ | |
| - readv = LOAD_I32(maddr); \ | |
| - STORE_U32(maddr, readv op sval); \ | |
| - shared_memory_unlock(memory); \ | |
| + ATOMIC_RMW(uint32, op, readv); \ | |
| } \ | |
| PUSH_I32(readv); \ | |
| break; \ | |
| @@ -799,39 +813,26 @@ wasm_interp_get_frame_ref(WASMInterpFrame *frame) | |
| CHECK_MEMORY_OVERFLOW(1); \ | |
| CHECK_ATOMIC_MEMORY_ACCESS(1); \ | |
| \ | |
| - shared_memory_lock(memory); \ | |
| - readv = (uint64)(*(uint8 *)maddr); \ | |
| - *(uint8 *)maddr = (uint8)(readv op sval); \ | |
| - shared_memory_unlock(memory); \ | |
| + ATOMIC_RMW(uint8, op, readv); \ | |
| } \ | |
| else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \ | |
| CHECK_MEMORY_OVERFLOW(2); \ | |
| CHECK_ATOMIC_MEMORY_ACCESS(2); \ | |
| \ | |
| - shared_memory_lock(memory); \ | |
| - readv = (uint64)LOAD_U16(maddr); \ | |
| - STORE_U16(maddr, (uint16)(readv op sval)); \ | |
| - shared_memory_unlock(memory); \ | |
| + ATOMIC_RMW(uint16, op, readv); \ | |
| } \ | |
| else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \ | |
| CHECK_MEMORY_OVERFLOW(4); \ | |
| CHECK_ATOMIC_MEMORY_ACCESS(4); \ | |
| \ | |
| - shared_memory_lock(memory); \ | |
| - readv = (uint64)LOAD_U32(maddr); \ | |
| - STORE_U32(maddr, (uint32)(readv op sval)); \ | |
| - shared_memory_unlock(memory); \ | |
| + ATOMIC_RMW(uint32, op, readv); \ | |
| } \ | |
| else { \ | |
| uint64 op_result; \ | |
| CHECK_MEMORY_OVERFLOW(8); \ | |
| CHECK_ATOMIC_MEMORY_ACCESS(8); \ | |
| \ | |
| - shared_memory_lock(memory); \ | |
| - readv = (uint64)LOAD_I64(maddr); \ | |
| - op_result = readv op sval; \ | |
| - STORE_I64(maddr, op_result); \ | |
| - shared_memory_unlock(memory); \ | |
| + ATOMIC_RMW(uint64, op, readv); \ | |
| } \ | |
| PUSH_I64(readv); \ | |
| break; \ | |
| @@ -6246,23 +6247,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, | |
| if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) { | |
| CHECK_MEMORY_OVERFLOW(1); | |
| CHECK_ATOMIC_MEMORY_ACCESS(1); | |
| - shared_memory_lock(memory); | |
| - readv = (uint32)(*(uint8 *)maddr); | |
| - shared_memory_unlock(memory); | |
| + readv = ATOMIC_LOAD(uint8); | |
| } | |
| else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) { | |
| CHECK_MEMORY_OVERFLOW(2); | |
| CHECK_ATOMIC_MEMORY_ACCESS(2); | |
| - shared_memory_lock(memory); | |
| - readv = (uint32)LOAD_U16(maddr); | |
| - shared_memory_unlock(memory); | |
| + readv = ATOMIC_LOAD(uint16); | |
| } | |
| else { | |
| CHECK_MEMORY_OVERFLOW(4); | |
| CHECK_ATOMIC_MEMORY_ACCESS(4); | |
| - shared_memory_lock(memory); | |
| - readv = LOAD_I32(maddr); | |
| - shared_memory_unlock(memory); | |
| + readv = ATOMIC_LOAD(uint32); | |
| } | |
| PUSH_I32(readv); | |
| @@ -6281,30 +6276,22 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, | |
| if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) { | |
| CHECK_MEMORY_OVERFLOW(1); | |
| CHECK_ATOMIC_MEMORY_ACCESS(1); | |
| - shared_memory_lock(memory); | |
| - readv = (uint64)(*(uint8 *)maddr); | |
| - shared_memory_unlock(memory); | |
| + readv = ATOMIC_LOAD(uint8); | |
| } | |
| else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) { | |
| CHECK_MEMORY_OVERFLOW(2); | |
| CHECK_ATOMIC_MEMORY_ACCESS(2); | |
| - shared_memory_lock(memory); | |
| - readv = (uint64)LOAD_U16(maddr); | |
| - shared_memory_unlock(memory); | |
| + readv = ATOMIC_LOAD(uint16); | |
| } | |
| else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) { | |
| CHECK_MEMORY_OVERFLOW(4); | |
| CHECK_ATOMIC_MEMORY_ACCESS(4); | |
| - shared_memory_lock(memory); | |
| - readv = (uint64)LOAD_U32(maddr); | |
| - shared_memory_unlock(memory); | |
| + readv = ATOMIC_LOAD(uint32); | |
| } | |
| else { | |
| CHECK_MEMORY_OVERFLOW(8); | |
| CHECK_ATOMIC_MEMORY_ACCESS(8); | |
| - shared_memory_lock(memory); | |
| - readv = LOAD_I64(maddr); | |
| - shared_memory_unlock(memory); | |
| + readv = ATOMIC_LOAD(uint64); | |
| } | |
| PUSH_I64(readv); | |
| @@ -6322,23 +6309,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, | |
| if (opcode == WASM_OP_ATOMIC_I32_STORE8) { | |
| CHECK_MEMORY_OVERFLOW(1); | |
| CHECK_ATOMIC_MEMORY_ACCESS(1); | |
| - shared_memory_lock(memory); | |
| - *(uint8 *)maddr = (uint8)sval; | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_STORE(uint8, sval); | |
| } | |
| else if (opcode == WASM_OP_ATOMIC_I32_STORE16) { | |
| CHECK_MEMORY_OVERFLOW(2); | |
| CHECK_ATOMIC_MEMORY_ACCESS(2); | |
| - shared_memory_lock(memory); | |
| - STORE_U16(maddr, (uint16)sval); | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_STORE(uint16, sval); | |
| } | |
| else { | |
| CHECK_MEMORY_OVERFLOW(4); | |
| CHECK_ATOMIC_MEMORY_ACCESS(4); | |
| - shared_memory_lock(memory); | |
| - STORE_U32(maddr, sval); | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_STORE(uint32, sval); | |
| } | |
| break; | |
| } | |
| @@ -6356,30 +6337,22 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, | |
| if (opcode == WASM_OP_ATOMIC_I64_STORE8) { | |
| CHECK_MEMORY_OVERFLOW(1); | |
| CHECK_ATOMIC_MEMORY_ACCESS(1); | |
| - shared_memory_lock(memory); | |
| - *(uint8 *)maddr = (uint8)sval; | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_STORE(uint8, sval); | |
| } | |
| else if (opcode == WASM_OP_ATOMIC_I64_STORE16) { | |
| CHECK_MEMORY_OVERFLOW(2); | |
| CHECK_ATOMIC_MEMORY_ACCESS(2); | |
| - shared_memory_lock(memory); | |
| - STORE_U16(maddr, (uint16)sval); | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_STORE(uint16, sval); | |
| } | |
| else if (opcode == WASM_OP_ATOMIC_I64_STORE32) { | |
| CHECK_MEMORY_OVERFLOW(4); | |
| CHECK_ATOMIC_MEMORY_ACCESS(4); | |
| - shared_memory_lock(memory); | |
| - STORE_U32(maddr, (uint32)sval); | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_STORE(uint32, sval); | |
| } | |
| else { | |
| CHECK_MEMORY_OVERFLOW(8); | |
| CHECK_ATOMIC_MEMORY_ACCESS(8); | |
| - shared_memory_lock(memory); | |
| - STORE_I64(maddr, sval); | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_STORE(uint64, sval); | |
| } | |
| break; | |
| } | |
| @@ -6399,32 +6372,20 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, | |
| CHECK_ATOMIC_MEMORY_ACCESS(1); | |
| expect = (uint8)expect; | |
| - shared_memory_lock(memory); | |
| - readv = (uint32)(*(uint8 *)maddr); | |
| - if (readv == expect) | |
| - *(uint8 *)maddr = (uint8)(sval); | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_CMPXCHG(uint8, expect, sval, readv); | |
| } | |
| else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) { | |
| CHECK_MEMORY_OVERFLOW(2); | |
| CHECK_ATOMIC_MEMORY_ACCESS(2); | |
| expect = (uint16)expect; | |
| - shared_memory_lock(memory); | |
| - readv = (uint32)LOAD_U16(maddr); | |
| - if (readv == expect) | |
| - STORE_U16(maddr, (uint16)(sval)); | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_CMPXCHG(uint16, expect, sval, readv); | |
| } | |
| else { | |
| CHECK_MEMORY_OVERFLOW(4); | |
| CHECK_ATOMIC_MEMORY_ACCESS(4); | |
| - shared_memory_lock(memory); | |
| - readv = LOAD_I32(maddr); | |
| - if (readv == expect) | |
| - STORE_U32(maddr, sval); | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_CMPXCHG(uint32, expect, sval, readv); | |
| } | |
| PUSH_I32(readv); | |
| break; | |
| @@ -6445,43 +6406,27 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, | |
| CHECK_ATOMIC_MEMORY_ACCESS(1); | |
| expect = (uint8)expect; | |
| - shared_memory_lock(memory); | |
| - readv = (uint64)(*(uint8 *)maddr); | |
| - if (readv == expect) | |
| - *(uint8 *)maddr = (uint8)(sval); | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_CMPXCHG(uint8, expect, sval, readv); | |
| } | |
| else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) { | |
| CHECK_MEMORY_OVERFLOW(2); | |
| CHECK_ATOMIC_MEMORY_ACCESS(2); | |
| expect = (uint16)expect; | |
| - shared_memory_lock(memory); | |
| - readv = (uint64)LOAD_U16(maddr); | |
| - if (readv == expect) | |
| - STORE_U16(maddr, (uint16)(sval)); | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_CMPXCHG(uint16, expect, sval, readv); | |
| } | |
| else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) { | |
| CHECK_MEMORY_OVERFLOW(4); | |
| CHECK_ATOMIC_MEMORY_ACCESS(4); | |
| expect = (uint32)expect; | |
| - shared_memory_lock(memory); | |
| - readv = (uint64)LOAD_U32(maddr); | |
| - if (readv == expect) | |
| - STORE_U32(maddr, (uint32)(sval)); | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_CMPXCHG(uint32, expect, sval, readv); | |
| } | |
| else { | |
| CHECK_MEMORY_OVERFLOW(8); | |
| CHECK_ATOMIC_MEMORY_ACCESS(8); | |
| - shared_memory_lock(memory); | |
| - readv = (uint64)LOAD_I64(maddr); | |
| - if (readv == expect) | |
| - STORE_I64(maddr, sval); | |
| - shared_memory_unlock(memory); | |
| + ATOMIC_CMPXCHG(uint64, expect, sval, readv); | |
| } | |
| PUSH_I64(readv); | |
| break; | |
| diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c | |
| index 819b266..6e0a43f 100644 | |
| --- a/core/iwasm/interpreter/wasm_runtime.c | |
| +++ b/core/iwasm/interpreter/wasm_runtime.c | |
| @@ -1684,6 +1684,11 @@ execute_post_instantiate_functions(WASMModuleInstance *module_inst, | |
| } | |
| #endif | |
| + /* rive: a parallel call member has no exec env to run them on, and the | |
| + host copies in the globals they would have set */ | |
| + if (is_sub_inst && !exec_env_main) | |
| + return true; | |
| + | |
| if (!start_func && !initialize_func && !post_inst_func | |
| && !call_ctors_func) { | |
| /* No post instantiation functions to call */ |