| diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c |
| index 5c78d30..c2aebd3 100644 |
| --- a/core/iwasm/interpreter/wasm_interp_fast.c |
| +++ b/core/iwasm/interpreter/wasm_interp_fast.c |
| @@ -1906,6 +1906,94 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, |
| HANDLE_OP_END(); |
| } |
| |
| +/* rive: fused arith/load pairs; operand layout keeps the producer's |
| + [rhs][lhs] in place, then the second op's operand, then dst */ |
| +#define DEF_OP_FUSED_ARITH(op1_expr, op2_expr) \ |
| + do { \ |
| + uint32 t = (op1_expr); \ |
| + SET_OPERAND(I32, 6, (op2_expr)); \ |
| + frame_ip += 8; \ |
| + } while (0) |
| + |
| + HANDLE_OP(EXT_OP_I32_SHL_ADD) |
| + { |
| + DEF_OP_FUSED_ARITH(GET_OPERAND(uint32, I32, 2) |
| + << (GET_OPERAND(uint32, I32, 0) % 32), |
| + GET_OPERAND(uint32, I32, 4) + t); |
| + HANDLE_OP_END(); |
| + } |
| + |
| + HANDLE_OP(EXT_OP_I32_SHR_U_AND) |
| + { |
| + DEF_OP_FUSED_ARITH(GET_OPERAND(uint32, I32, 2) |
| + >> (GET_OPERAND(uint32, I32, 0) % 32), |
| + GET_OPERAND(uint32, I32, 4) & t); |
| + HANDLE_OP_END(); |
| + } |
| + |
| + HANDLE_OP(EXT_OP_I32_AND_ADD) |
| + { |
| + DEF_OP_FUSED_ARITH(GET_OPERAND(uint32, I32, 2) |
| + & GET_OPERAND(uint32, I32, 0), |
| + GET_OPERAND(uint32, I32, 4) + t); |
| + HANDLE_OP_END(); |
| + } |
| + |
| + HANDLE_OP(EXT_OP_I32_ADD_SHR_U) |
| + { |
| + DEF_OP_FUSED_ARITH(GET_OPERAND(uint32, I32, 2) |
| + + GET_OPERAND(uint32, I32, 0), |
| + t >> (GET_OPERAND(uint32, I32, 4) % 32)); |
| + HANDLE_OP_END(); |
| + } |
| + |
| + HANDLE_OP(EXT_OP_I32_AND_SHL) |
| + { |
| + DEF_OP_FUSED_ARITH(GET_OPERAND(uint32, I32, 2) |
| + & GET_OPERAND(uint32, I32, 0), |
| + t << (GET_OPERAND(uint32, I32, 4) % 32)); |
| + HANDLE_OP_END(); |
| + } |
| + |
| + HANDLE_OP(EXT_OP_I32_LOAD_ADD) |
| + { |
| + uint32 offset, addr; |
| + addr = |
| + GET_OPERAND(uint32, I32, 2) + GET_OPERAND(uint32, I32, 0); |
| + frame_ip += 4; |
| + offset = read_uint32(frame_ip); |
| + addr_ret = GET_OFFSET(); |
| + CHECK_MEMORY_OVERFLOW(4); |
| + frame_lp[addr_ret] = LOAD_I32(maddr); |
| + HANDLE_OP_END(); |
| + } |
| + |
| + HANDLE_OP(EXT_OP_I32_LOAD_SHR_U) |
| + { |
| + uint32 offset, addr, shift; |
| + offset = read_uint32(frame_ip); |
| + addr = GET_OPERAND(uint32, I32, 0); |
| + shift = GET_OPERAND(uint32, I32, 2); |
| + frame_ip += 4; |
| + addr_ret = GET_OFFSET(); |
| + CHECK_MEMORY_OVERFLOW(4); |
| + frame_lp[addr_ret] = (uint32)LOAD_I32(maddr) >> (shift % 32); |
| + HANDLE_OP_END(); |
| + } |
| + |
| + HANDLE_OP(EXT_OP_I32_LOAD8_U_SHL) |
| + { |
| + uint32 offset, addr, shift; |
| + offset = read_uint32(frame_ip); |
| + addr = GET_OPERAND(uint32, I32, 0); |
| + shift = GET_OPERAND(uint32, I32, 2); |
| + frame_ip += 4; |
| + addr_ret = GET_OFFSET(); |
| + CHECK_MEMORY_OVERFLOW(1); |
| + frame_lp[addr_ret] = (uint32)(*(uint8 *)maddr) << (shift % 32); |
| + HANDLE_OP_END(); |
| + } |
| + |
| HANDLE_OP(EXT_OP_BR_IF_I32_GE_U) |
| { |
| #if WASM_ENABLE_THREAD_MGR != 0 |
| diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c |
| index 8cf82b0..3f64472 100644 |
| --- a/core/iwasm/interpreter/wasm_loader.c |
| +++ b/core/iwasm/interpreter/wasm_loader.c |
| @@ -9255,6 +9255,15 @@ fail: |
| #define WASM_LOADER_FUSE_BR 0 |
| #endif |
| |
| +/* rive: arith/load pair fusion patches emitted labels in place, which |
| + needs the pointer-sized label encoding */ |
| +#if WASM_LOADER_FUSE_BR != 0 && WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0 \ |
| + && WASM_ENABLE_MEMORY64 == 0 |
| +#define WASM_LOADER_FUSE_ARITH 1 |
| +#else |
| +#define WASM_LOADER_FUSE_ARITH 0 |
| +#endif |
| + |
| #define emit_br_info(frame_csp, is_br, omit_zero_arity) \ |
| do { \ |
| if (!wasm_loader_emit_br_info(loader_ctx, frame_csp, is_br, \ |
| @@ -9590,6 +9599,20 @@ preserve_referenced_local(WASMLoaderContext *loader_ctx, uint8 opcode, |
| return true; |
| } |
| |
| +#if WASM_LOADER_FUSE_ARITH != 0 |
| +/* pop offsets without emitting them, for operands a fused op absorbed */ |
| +static void |
| +wasm_loader_fuse_pop_offsets(WASMLoaderContext *ctx, uint32 cells) |
| +{ |
| + while (cells--) { |
| + ctx->frame_offset--; |
| + if ((*ctx->frame_offset > ctx->start_dynamic_offset) |
| + && (*ctx->frame_offset < ctx->max_dynamic_offset)) |
| + ctx->dynamic_offset--; |
| + } |
| +} |
| +#endif |
| + |
| static bool |
| preserve_local_for_block(WASMLoaderContext *loader_ctx, uint8 opcode, |
| char *error_buf, uint32 error_buf_size) |
| @@ -11670,6 +11693,13 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, |
| #if WASM_ENABLE_FAST_INTERP != 0 |
| int16 operand_offset = 0; |
| uint8 last_op = 0; |
| +#if WASM_LOADER_FUSE_ARITH != 0 |
| + /* pending fusable producer: its opcode, dst slot, and how many single |
| + cell pushes (consts / local gets) landed above it since */ |
| + uint8 fuse_prod_op = 0, fuse_prod_depth = 0; |
| + int16 fuse_prod_off = 0; |
| + bool fuse_suppress = false; |
| +#endif |
| bool disable_emit, preserve_local = false, if_condition_available = true; |
| float32 f32_const; |
| float64 f64_const; |
| @@ -14358,6 +14388,32 @@ re_scan: |
| goto fail; |
| } |
| #if WASM_ENABLE_FAST_INTERP != 0 |
| +#if WASM_LOADER_FUSE_ARITH != 0 |
| + if (opcode == WASM_OP_I32_LOAD |
| + && fuse_prod_op == WASM_OP_I32_ADD && fuse_prod_depth == 0 |
| + && !(loader_ctx->frame_csp - 1)->is_stack_polymorphic |
| + && loader_ctx->stack_cell_num |
| + > (loader_ctx->frame_csp - 1)->stack_cell_num |
| + && *(loader_ctx->frame_offset - 1) == fuse_prod_off) { |
| + skip_label(); |
| + /* take over the add's instruction: drop its dst and |
| + append the memarg in place */ |
| + wasm_loader_emit_backspace(loader_ctx, 2); |
| + if (loader_ctx->p_code_compiled) |
| + *(void **)(loader_ctx->p_code_compiled - 4 |
| + - sizeof(void *)) = |
| + handle_table[EXT_OP_I32_LOAD_ADD]; |
| + emit_uint32(loader_ctx, mem_offset); |
| + wasm_loader_fuse_pop_offsets(loader_ctx, 1); |
| + POP_TYPE(VALUE_TYPE_I32); |
| + PUSH_I32(); |
| + fuse_suppress = true; |
| +#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 |
| + func->has_memory_operations = true; |
| +#endif |
| + break; |
| + } |
| +#endif |
| emit_uint32(loader_ctx, mem_offset); |
| #endif |
| #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 |
| @@ -14589,8 +14645,79 @@ re_scan: |
| case WASM_OP_I32_SHR_U: |
| case WASM_OP_I32_ROTL: |
| case WASM_OP_I32_ROTR: |
| +#if WASM_LOADER_FUSE_ARITH != 0 |
| + { |
| + uint8 fused_arith = 0; |
| + if (fuse_prod_op |
| + && !(loader_ctx->frame_csp - 1)->is_stack_polymorphic |
| + && loader_ctx->stack_cell_num |
| + - (loader_ctx->frame_csp - 1)->stack_cell_num |
| + >= 2 |
| + && *(loader_ctx->frame_offset - 1 - fuse_prod_depth) |
| + == fuse_prod_off) { |
| + switch (opcode) { |
| + case WASM_OP_I32_ADD: |
| + if (fuse_prod_op == WASM_OP_I32_SHL) |
| + fused_arith = EXT_OP_I32_SHL_ADD; |
| + else if (fuse_prod_op == WASM_OP_I32_AND) |
| + fused_arith = EXT_OP_I32_AND_ADD; |
| + break; |
| + case WASM_OP_I32_AND: |
| + if (fuse_prod_op == WASM_OP_I32_SHR_U) |
| + fused_arith = EXT_OP_I32_SHR_U_AND; |
| + break; |
| + /* non-commutative consumers only fuse a producer in |
| + lhs position */ |
| + case WASM_OP_I32_SHR_U: |
| + if (fuse_prod_depth == 1) { |
| + if (fuse_prod_op == WASM_OP_I32_ADD) |
| + fused_arith = EXT_OP_I32_ADD_SHR_U; |
| + else if (fuse_prod_op == WASM_OP_I32_LOAD) |
| + fused_arith = EXT_OP_I32_LOAD_SHR_U; |
| + } |
| + break; |
| + case WASM_OP_I32_SHL: |
| + if (fuse_prod_depth == 1) { |
| + if (fuse_prod_op == WASM_OP_I32_AND) |
| + fused_arith = EXT_OP_I32_AND_SHL; |
| + else if (fuse_prod_op == WASM_OP_I32_LOAD8_U) |
| + fused_arith = EXT_OP_I32_LOAD8_U_SHL; |
| + } |
| + break; |
| + } |
| + } |
| + if (fused_arith) { |
| + uint32 prod_bytes = (fuse_prod_op == WASM_OP_I32_LOAD |
| + || fuse_prod_op == WASM_OP_I32_LOAD8_U) |
| + ? 8 |
| + : 6; |
| + skip_label(); |
| + if (loader_ctx->p_code_compiled) { |
| + /* take over the producer's instruction: retarget its |
| + label and turn its dst slot into the second |
| + operand of the fused op */ |
| + int16 other_off = |
| + *(loader_ctx->frame_offset - 2 + fuse_prod_depth); |
| + *(void **)(loader_ctx->p_code_compiled - prod_bytes |
| + - sizeof(void *)) = |
| + handle_table[fused_arith]; |
| + STORE_U16(loader_ctx->p_code_compiled - 2, |
| + (uint16)other_off); |
| + } |
| + wasm_loader_fuse_pop_offsets(loader_ctx, 2); |
| + POP_TYPE(VALUE_TYPE_I32); |
| + POP_TYPE(VALUE_TYPE_I32); |
| + PUSH_I32(); |
| + fuse_suppress = true; |
| + break; |
| + } |
| + POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); |
| + break; |
| + } |
| +#else |
| POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); |
| break; |
| +#endif |
| |
| case WASM_OP_I64_CLZ: |
| case WASM_OP_I64_CTZ: |
| @@ -16996,6 +17123,46 @@ re_scan: |
| |
| #if WASM_ENABLE_FAST_INTERP != 0 |
| last_op = opcode; |
| +#endif |
| +#if WASM_LOADER_FUSE_ARITH != 0 |
| + if (fuse_suppress) { |
| + /* an op fused this iteration left a non-standard encoding, so |
| + its result must not seed another fusion */ |
| + fuse_suppress = false; |
| + fuse_prod_op = 0; |
| + } |
| + else { |
| + switch (opcode) { |
| + case WASM_OP_I32_ADD: |
| + case WASM_OP_I32_AND: |
| + case WASM_OP_I32_SHL: |
| + case WASM_OP_I32_SHR_U: |
| + case WASM_OP_I32_LOAD: |
| + case WASM_OP_I32_LOAD8_U: |
| + if (!(loader_ctx->frame_csp - 1)->is_stack_polymorphic |
| + && loader_ctx->stack_cell_num |
| + > (loader_ctx->frame_csp - 1)->stack_cell_num) { |
| + fuse_prod_op = opcode; |
| + fuse_prod_depth = 0; |
| + fuse_prod_off = *(loader_ctx->frame_offset - 1); |
| + } |
| + else |
| + fuse_prod_op = 0; |
| + break; |
| + case WASM_OP_I32_CONST: |
| + case WASM_OP_GET_LOCAL: |
| + if (fuse_prod_depth == 0 |
| + && (opcode == WASM_OP_I32_CONST |
| + || is_32bit_type(local_type))) |
| + fuse_prod_depth = 1; |
| + else |
| + fuse_prod_op = 0; |
| + break; |
| + default: |
| + fuse_prod_op = 0; |
| + break; |
| + } |
| + } |
| #endif |
| } |
| |
| diff --git a/core/iwasm/interpreter/wasm_opcode.h b/core/iwasm/interpreter/wasm_opcode.h |
| index 217c0e0..1a0b3a3 100644 |
| --- a/core/iwasm/interpreter/wasm_opcode.h |
| +++ b/core/iwasm/interpreter/wasm_opcode.h |
| @@ -303,6 +303,17 @@ typedef enum WASMOpcode { |
| EXT_OP_BR_IF_I32_GE_S = 0xee, |
| EXT_OP_BR_IF_I32_GE_U = 0xef, |
| |
| + /* rive: profile-guided arith/load pair fusions, emitted only by the |
| + fast-interp loader */ |
| + EXT_OP_I32_SHL_ADD = 0xf0, /* dst = c + (a << b) */ |
| + EXT_OP_I32_SHR_U_AND = 0xf1, /* dst = c & (a >> b) */ |
| + EXT_OP_I32_AND_ADD = 0xf2, /* dst = c + (a & b) */ |
| + EXT_OP_I32_ADD_SHR_U = 0xf3, /* dst = (a + b) >> c */ |
| + EXT_OP_I32_AND_SHL = 0xf4, /* dst = (a & b) << c */ |
| + EXT_OP_I32_LOAD_ADD = 0xf5, /* dst = mem32[moff + a + b] */ |
| + EXT_OP_I32_LOAD_SHR_U = 0xf6, /* dst = mem32[moff + addr] >> c */ |
| + EXT_OP_I32_LOAD8_U_SHL = 0xf7, /* dst = mem8[moff + addr] << c */ |
| + |
| /* Post-MVP extend op prefix */ |
| WASM_OP_GC_PREFIX = 0xfb, |
| WASM_OP_MISC_PREFIX = 0xfc, |
| @@ -832,7 +843,15 @@ typedef enum WASMAtomicEXTOpcode { |
| SET_GOTO_TABLE_ELEM(EXT_OP_BR_IF_I32_LE_S), \ |
| SET_GOTO_TABLE_ELEM(EXT_OP_BR_IF_I32_LE_U), \ |
| SET_GOTO_TABLE_ELEM(EXT_OP_BR_IF_I32_GE_S), \ |
| - SET_GOTO_TABLE_ELEM(EXT_OP_BR_IF_I32_GE_U), |
| + SET_GOTO_TABLE_ELEM(EXT_OP_BR_IF_I32_GE_U), \ |
| + SET_GOTO_TABLE_ELEM(EXT_OP_I32_SHL_ADD), \ |
| + SET_GOTO_TABLE_ELEM(EXT_OP_I32_SHR_U_AND), \ |
| + SET_GOTO_TABLE_ELEM(EXT_OP_I32_AND_ADD), \ |
| + SET_GOTO_TABLE_ELEM(EXT_OP_I32_ADD_SHR_U), \ |
| + SET_GOTO_TABLE_ELEM(EXT_OP_I32_AND_SHL), \ |
| + SET_GOTO_TABLE_ELEM(EXT_OP_I32_LOAD_ADD), \ |
| + SET_GOTO_TABLE_ELEM(EXT_OP_I32_LOAD_SHR_U), \ |
| + SET_GOTO_TABLE_ELEM(EXT_OP_I32_LOAD8_U_SHL), |
| #else |
| #define DEF_EXT_FUSED_BR_HANDLES() |
| #endif |
