| --- a/core/iwasm/common/wasm_runtime_common.h | |
| +++ b/core/iwasm/common/wasm_runtime_common.h | |
| @@ -1393,6 +1393,30 @@ | |
| WASM_RUNTIME_API_EXTERN bool | |
| wasm_runtime_detect_native_stack_overflow(WASMExecEnv *exec_env); | |
| + | |
| +/* rive: the one body of the native stack check, inline so the interpreter's | |
| + host calls skip a function call; the exported function forwards here */ | |
| +static inline bool | |
| +wasm_runtime_detect_native_stack_overflow_inline(WASMExecEnv *exec_env) | |
| +{ | |
| + uint8 *boundary = exec_env->native_stack_boundary; | |
| + RECORD_STACK_USAGE(exec_env, (uint8 *)&boundary); | |
| + if (boundary == NULL) { | |
| + /* the platform doesn't support os_thread_get_stack_boundary */ | |
| + return true; | |
| + } | |
| +#if defined(OS_ENABLE_HW_BOUND_CHECK) && WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 | |
| + uint32 page_size = os_getpagesize(); | |
| + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; | |
| + boundary = boundary + page_size * guard_page_count; | |
| +#endif | |
| + if ((uint8 *)&boundary < boundary) { | |
| + wasm_runtime_set_exception(exec_env->module_inst, | |
| + "native stack overflow"); | |
| + return false; | |
| + } | |
| + return true; | |
| +} | |
| WASM_RUNTIME_API_EXTERN bool | |
| wasm_runtime_detect_native_stack_overflow_size(WASMExecEnv *exec_env, | |
| --- a/core/iwasm/common/wasm_runtime_common.c | |
| +++ b/core/iwasm/common/wasm_runtime_common.c | |
| @@ -3268,6 +3268,8 @@ | |
| || module_inst_comm->module_type == Wasm_Module_AoT); | |
| wasm_runtime_set_exception(module_inst_comm, NULL); | |
| } | |
| + | |
| +const char wasm_runtime_rive_leaf_native = 0; | |
| void | |
| wasm_runtime_rive_interrupt(WASMExecEnv *exec_env) | |
| @@ -6371,7 +6373,9 @@ | |
| void *attachment, uint32 *argv, | |
| uint32 *argv_ret) | |
| { | |
| - WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); | |
| + WASMModuleInstanceCommon *module = exec_env->module_inst; | |
| + const uint8 *types = func_type->types; | |
| + uint32 param_count = func_type->param_count; | |
| uint64 ints[4] = { 0 }; | |
| double fps[8] = { 0 }; | |
| union { | |
| @@ -6380,6 +6384,58 @@ | |
| } fbits; | |
| uint32 n_int = 0, n_fp = 0, i; | |
| uint32 *p = argv; | |
| + | |
| + /* rive: exact-signature calls for the binding shapes that dominate, | |
| + skipping the zeroed register file the general path below passes */ | |
| + if (param_count <= 3 | |
| + && (ret_kind == WASM_QUICK_NATIVE_RET_VOID | |
| + || ret_kind == WASM_QUICK_NATIVE_RET_I32 | |
| + || ret_kind == WASM_QUICK_NATIVE_RET_F32)) { | |
| + bool all_i32 = true; | |
| + for (i = 0; i < param_count; i++) { | |
| + if (types[i] != VALUE_TYPE_I32) | |
| + all_i32 = false; | |
| + } | |
| + bool i32_f32 = param_count == 2 && types[0] == VALUE_TYPE_I32 | |
| + && types[1] == VALUE_TYPE_F32; | |
| + bool shaped = ret_kind == WASM_QUICK_NATIVE_RET_F32 | |
| + ? all_i32 && param_count == 1 | |
| + : all_i32 || i32_f32; | |
| + if (shaped) { | |
| + float32 f32_arg = 0; | |
| + if (i32_f32) | |
| + memcpy(&f32_arg, &argv[1], sizeof(f32_arg)); | |
| + /* the pointer type matches the native's, so void natives are | |
| + never called through a value returning type */ | |
| +#define QUICK_SHAPED_CALL(R) \ | |
| + (i32_f32 ? ((R(*)(WASMExecEnv *, uint32, float32))func_ptr)( \ | |
| + exec_env, argv[0], f32_arg) \ | |
| + : param_count == 0 ? ((R(*)(WASMExecEnv *))func_ptr)(exec_env) \ | |
| + : param_count == 1 \ | |
| + ? ((R(*)(WASMExecEnv *, uint32))func_ptr)(exec_env, argv[0]) \ | |
| + : param_count == 2 \ | |
| + ? ((R(*)(WASMExecEnv *, uint32, uint32))func_ptr)(exec_env, argv[0], \ | |
| + argv[1]) \ | |
| + : ((R(*)(WASMExecEnv *, uint32, uint32, uint32))func_ptr)( \ | |
| + exec_env, argv[0], argv[1], argv[2])) | |
| + exec_env->attachment = attachment; | |
| + if (ret_kind == WASM_QUICK_NATIVE_RET_F32) { | |
| + *(float32 *)argv_ret = | |
| + ((float32(*)(WASMExecEnv *, uint32))func_ptr)(exec_env, | |
| + argv[0]); | |
| + } | |
| + else if (ret_kind == WASM_QUICK_NATIVE_RET_VOID) { | |
| + QUICK_SHAPED_CALL(void); | |
| + } | |
| + else { | |
| + argv_ret[0] = QUICK_SHAPED_CALL(uint32); | |
| + } | |
| +#undef QUICK_SHAPED_CALL | |
| + exec_env->attachment = NULL; | |
| + return !wasm_runtime_has_exception_inline( | |
| + (WASMModuleInstance *)module); | |
| + } | |
| + } | |
| for (i = 0; i < func_type->param_count; i++) { | |
| switch (func_type->types[i]) { | |
| @@ -6427,7 +6483,7 @@ | |
| } | |
| exec_env->attachment = NULL; | |
| - return !wasm_runtime_copy_exception(module, NULL); | |
| + return !wasm_runtime_has_exception_inline((WASMModuleInstance *)module); | |
| } | |
| #endif /* end of WASM_ENABLE_QUICK_NATIVE */ | |
| @@ -8067,23 +8123,7 @@ | |
| bool | |
| wasm_runtime_detect_native_stack_overflow(WASMExecEnv *exec_env) | |
| { | |
| - uint8 *boundary = exec_env->native_stack_boundary; | |
| - RECORD_STACK_USAGE(exec_env, (uint8 *)&boundary); | |
| - if (boundary == NULL) { | |
| - /* the platform doesn't support os_thread_get_stack_boundary */ | |
| - return true; | |
| - } | |
| -#if defined(OS_ENABLE_HW_BOUND_CHECK) && WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 | |
| - uint32 page_size = os_getpagesize(); | |
| - uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; | |
| - boundary = boundary + page_size * guard_page_count; | |
| -#endif | |
| - if ((uint8 *)&boundary < boundary) { | |
| - wasm_runtime_set_exception(wasm_runtime_get_module_inst(exec_env), | |
| - "native stack overflow"); | |
| - return false; | |
| - } | |
| - return true; | |
| + return wasm_runtime_detect_native_stack_overflow_inline(exec_env); | |
| } | |
| bool | |
| --- a/core/iwasm/interpreter/wasm_runtime.h | |
| +++ b/core/iwasm/interpreter/wasm_runtime.h | |
| @@ -894,6 +894,18 @@ | |
| #define exception_lock(module_inst) (void)(module_inst) | |
| #define exception_unlock(module_inst) (void)(module_inst) | |
| #endif | |
| + | |
| +/* rive: wasm_copy_exception's test without the call, under the same lock; | |
| + the host call path checks this after every native */ | |
| +static inline bool | |
| +wasm_runtime_has_exception_inline(WASMModuleInstance *module_inst) | |
| +{ | |
| + bool has_exception; | |
| + exception_lock(module_inst); | |
| + has_exception = module_inst->cur_exception[0] != '\0'; | |
| + exception_unlock(module_inst); | |
| + return has_exception; | |
| +} | |
| bool | |
| wasm_check_utf8_str(const uint8 *str, uint32 len); | |
| --- a/core/iwasm/interpreter/wasm_interp_fast.c | |
| +++ b/core/iwasm/interpreter/wasm_interp_fast.c | |
| @@ -1320,13 +1320,22 @@ | |
| unsigned local_cell_num = | |
| cur_func->param_cell_num > 2 ? cur_func->param_cell_num : 2; | |
| unsigned all_cell_num; | |
| - WASMInterpFrame *frame; | |
| + WASMInterpFrame *frame = NULL; | |
| uint32 argv_ret[2], cur_func_index; | |
| + uint32 *lp; | |
| void *native_func_pointer = NULL; | |
| bool ret; | |
| #if WASM_ENABLE_GC != 0 | |
| WASMFuncType *func_type; | |
| uint8 *frame_ref; | |
| +#endif | |
| + /* rive: a leaf never re-enters a module, so it reads the arguments the | |
| + caller staged at the stack top and needs no frame of its own */ | |
| +#if WASM_ENABLE_GC == 0 | |
| + bool leaf = func_import->attachment == &wasm_runtime_rive_leaf_native | |
| + && !func_import->call_conv_wasm_c_api; | |
| +#else | |
| + bool leaf = false; | |
| #endif | |
| all_cell_num = local_cell_num; | |
| @@ -1334,24 +1343,31 @@ | |
| all_cell_num += (local_cell_num + 3) / 4; | |
| #endif | |
| - if (!wasm_runtime_detect_native_stack_overflow(exec_env)) { | |
| + if (!wasm_runtime_detect_native_stack_overflow_inline(exec_env)) { | |
| return; | |
| } | |
| - if (!(frame = | |
| - ALLOC_FRAME(exec_env, wasm_interp_interp_frame_size(all_cell_num), | |
| - prev_frame))) | |
| - return; | |
| - | |
| - frame->function = cur_func; | |
| - frame->ip = NULL; | |
| - frame->lp = frame->operand; | |
| + if (leaf) { | |
| + lp = ((WASMInterpFrame *)wasm_exec_env_wasm_stack_top(exec_env)) | |
| + ->operand; | |
| + } | |
| + else { | |
| + if (!(frame = ALLOC_FRAME(exec_env, | |
| + wasm_interp_interp_frame_size(all_cell_num), | |
| + prev_frame))) | |
| + return; | |
| + | |
| + frame->function = cur_func; | |
| + frame->ip = NULL; | |
| + frame->lp = frame->operand; | |
| #if WASM_ENABLE_GC != 0 | |
| - frame->frame_ref = (uint8 *)(frame->lp + local_cell_num); | |
| - init_frame_refs(frame->frame_ref, local_cell_num, cur_func); | |
| + frame->frame_ref = (uint8 *)(frame->lp + local_cell_num); | |
| + init_frame_refs(frame->frame_ref, local_cell_num, cur_func); | |
| #endif | |
| - wasm_exec_env_set_cur_frame(exec_env, frame); | |
| + wasm_exec_env_set_cur_frame(exec_env, frame); | |
| + lp = frame->lp; | |
| + } | |
| cur_func_index = (uint32)(cur_func - module_inst->e->functions); | |
| bh_assert(cur_func_index < module_inst->module->import_function_count); | |
| @@ -1394,18 +1410,18 @@ | |
| if (quick_kind != WASM_QUICK_NATIVE_NO) | |
| ret = wasm_runtime_invoke_native_quick( | |
| exec_env, native_func_pointer, func_import->func_type, | |
| - quick_kind, func_import->attachment, frame->lp, argv_ret); | |
| + quick_kind, func_import->attachment, lp, argv_ret); | |
| else | |
| #endif | |
| ret = wasm_runtime_invoke_native( | |
| exec_env, native_func_pointer, func_import->func_type, | |
| - func_import->signature, func_import->attachment, frame->lp, | |
| + func_import->signature, func_import->attachment, lp, | |
| cur_func->param_cell_num, argv_ret); | |
| } | |
| else { | |
| ret = wasm_runtime_invoke_native_raw( | |
| exec_env, native_func_pointer, func_import->func_type, | |
| - func_import->signature, func_import->attachment, frame->lp, | |
| + func_import->signature, func_import->attachment, lp, | |
| cur_func->param_cell_num, argv_ret); | |
| } | |
| @@ -1434,8 +1450,10 @@ | |
| prev_frame->lp[prev_frame->ret_offset + 1] = argv_ret[1]; | |
| } | |
| - FREE_FRAME(exec_env, frame); | |
| - wasm_exec_env_set_cur_frame(exec_env, prev_frame); | |
| + if (frame != NULL) { | |
| + FREE_FRAME(exec_env, frame); | |
| + wasm_exec_env_set_cur_frame(exec_env, prev_frame); | |
| + } | |
| } | |
| #if WASM_ENABLE_MULTI_MODULE != 0 | |
| @@ -8084,7 +8102,7 @@ | |
| linear_mem_base = memory->memory_data; | |
| } | |
| #endif | |
| - if (wasm_copy_exception(module, NULL)) | |
| + if (wasm_runtime_has_exception_inline(module)) | |
| goto got_exception; | |
| } | |
| else { | |
| --- a/core/iwasm/include/wasm_export.h | |
| +++ b/core/iwasm/include/wasm_export.h | |
| @@ -1342,6 +1342,13 @@ | |
| wasm_runtime_rive_clear_interrupt(wasm_exec_env_t exec_env); | |
| /** | |
| + * rive: a native registered with &wasm_runtime_rive_leaf_native as its | |
| + * attachment promises never to call back into any module; the interpreter | |
| + * then runs it without a frame of its own. | |
| + */ | |
| +WASM_RUNTIME_API_EXTERN extern const char wasm_runtime_rive_leaf_native; | |
| + | |
| +/** | |
| * Set custom data to WASM module instance. | |
| * Note: | |
| * If WAMR_BUILD_LIB_PTHREAD is enabled, this API |