diff options
| author | Takashi Kokubun <[email protected]> | 2025-06-04 17:14:40 -0400 |
|---|---|---|
| committer | GitHub <[email protected]> | 2025-06-04 14:14:40 -0700 |
| commit | 112c34252d15d94bff87241ade18badc09923b09 (patch) | |
| tree | 2f7d64152bc57ee0521391d145ac557dda6ad7bd /zjit/src/codegen.rs | |
| parent | 99cc100cdf4f424faea8caa33beb0d14171c5dcb (diff) | |
ZJIT: Implement side exits for entry frames (#13469)
Co-authored-by: Max Bernstein <[email protected]>
Co-authored-by: Alan Wu <[email protected]>
Notes
Notes:
Merged-By: k0kubun <[email protected]>
Diffstat (limited to 'zjit/src/codegen.rs')
| -rw-r--r-- | zjit/src/codegen.rs | 53 |
1 files changed, 37 insertions, 16 deletions
diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index d5202486f1..c1fad915f0 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -260,9 +260,9 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::SendWithoutBlock { call_info, cd, state, self_val, args, .. } => gen_send_without_block(jit, asm, call_info, *cd, &function.frame_state(*state), self_val, args)?, Insn::SendWithoutBlockDirect { iseq, self_val, args, .. } => gen_send_without_block_direct(cb, jit, asm, *iseq, opnd!(self_val), args)?, Insn::Return { val } => return Some(gen_return(asm, opnd!(val))?), - Insn::FixnumAdd { left, right, state } => gen_fixnum_add(asm, opnd!(left), opnd!(right), &function.frame_state(*state))?, - Insn::FixnumSub { left, right, state } => gen_fixnum_sub(asm, opnd!(left), opnd!(right), &function.frame_state(*state))?, - Insn::FixnumMult { left, right, state } => gen_fixnum_mult(asm, opnd!(left), opnd!(right), &function.frame_state(*state))?, + Insn::FixnumAdd { left, right, state } => gen_fixnum_add(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state))?, + Insn::FixnumSub { left, right, state } => gen_fixnum_sub(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state))?, + Insn::FixnumMult { left, right, state } => gen_fixnum_mult(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state))?, Insn::FixnumEq { left, right } => gen_fixnum_eq(asm, opnd!(left), opnd!(right))?, Insn::FixnumNeq { left, right } => gen_fixnum_neq(asm, opnd!(left), opnd!(right))?, Insn::FixnumLt { left, right } => gen_fixnum_lt(asm, opnd!(left), opnd!(right))?, @@ -270,8 +270,8 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::FixnumGt { left, right } => gen_fixnum_gt(asm, opnd!(left), opnd!(right))?, Insn::FixnumGe { left, right } => gen_fixnum_ge(asm, opnd!(left), opnd!(right))?, Insn::Test { val } => gen_test(asm, opnd!(val))?, - Insn::GuardType { val, guard_type, state } => gen_guard_type(asm, opnd!(val), *guard_type, &function.frame_state(*state))?, - Insn::GuardBitEquals { val, expected, state } => gen_guard_bit_equals(asm, opnd!(val), *expected, &function.frame_state(*state))?, + Insn::GuardType { val, guard_type, state } => gen_guard_type(jit, asm, opnd!(val), *guard_type, &function.frame_state(*state))?, + Insn::GuardBitEquals { val, expected, state } => gen_guard_bit_equals(jit, asm, opnd!(val), *expected, &function.frame_state(*state))?, Insn::PatchPoint(_) => return Some(()), // For now, rb_zjit_bop_redefined() panics. TODO: leave a patch point and fix rb_zjit_bop_redefined() Insn::CCall { cfun, args, name: _, return_type: _, elidable: _ } => gen_ccall(jit, asm, *cfun, args)?, _ => { @@ -569,27 +569,27 @@ fn gen_return(asm: &mut Assembler, val: lir::Opnd) -> Option<()> { } /// Compile Fixnum + Fixnum -fn gen_fixnum_add(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> Option<lir::Opnd> { +fn gen_fixnum_add(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> Option<lir::Opnd> { // Add left + right and test for overflow let left_untag = asm.sub(left, Opnd::Imm(1)); let out_val = asm.add(left_untag, right); - asm.jo(Target::SideExit(state.clone())); + asm.jo(side_exit(jit, state)?); Some(out_val) } /// Compile Fixnum - Fixnum -fn gen_fixnum_sub(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> Option<lir::Opnd> { +fn gen_fixnum_sub(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> Option<lir::Opnd> { // Subtract left - right and test for overflow let val_untag = asm.sub(left, right); - asm.jo(Target::SideExit(state.clone())); + asm.jo(side_exit(jit, state)?); let out_val = asm.add(val_untag, Opnd::Imm(1)); Some(out_val) } /// Compile Fixnum * Fixnum -fn gen_fixnum_mult(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> Option<lir::Opnd> { +fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> Option<lir::Opnd> { // Do some bitwise gymnastics to handle tag bits // x * y is translated to (x >> 1) * (y - 1) + 1 let left_untag = asm.rshift(left, Opnd::UImm(1)); @@ -597,7 +597,7 @@ fn gen_fixnum_mult(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state let out_val = asm.mul(left_untag, right_untag); // Test for overflow - asm.jo_mul(Target::SideExit(state.clone())); + asm.jo_mul(side_exit(jit, state)?); let out_val = asm.add(out_val, Opnd::UImm(1)); Some(out_val) @@ -651,11 +651,11 @@ fn gen_test(asm: &mut Assembler, val: lir::Opnd) -> Option<lir::Opnd> { } /// Compile a type check with a side exit -fn gen_guard_type(asm: &mut Assembler, val: lir::Opnd, guard_type: Type, state: &FrameState) -> Option<lir::Opnd> { +fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard_type: Type, state: &FrameState) -> Option<lir::Opnd> { if guard_type.is_subtype(Fixnum) { // Check if opnd is Fixnum asm.test(val, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); - asm.jz(Target::SideExit(state.clone())); + asm.jz(side_exit(jit, state)?); } else { unimplemented!("unsupported type: {guard_type}"); } @@ -663,9 +663,9 @@ fn gen_guard_type(asm: &mut Assembler, val: lir::Opnd, guard_type: Type, state: } /// Compile an identity check with a side exit -fn gen_guard_bit_equals(asm: &mut Assembler, val: lir::Opnd, expected: VALUE, state: &FrameState) -> Option<lir::Opnd> { +fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, expected: VALUE, state: &FrameState) -> Option<lir::Opnd> { asm.cmp(val, Opnd::UImm(expected.into())); - asm.jnz(Target::SideExit(state.clone())); + asm.jnz(side_exit(jit, state)?); Some(val) } @@ -731,6 +731,26 @@ fn compile_iseq(iseq: IseqPtr) -> Option<Function> { Some(function) } +/// Build a Target::SideExit out of a FrameState +fn side_exit(jit: &mut JITState, state: &FrameState) -> Option<Target> { + let mut stack = Vec::new(); + for &insn_id in state.stack() { + stack.push(jit.get_opnd(insn_id)?); + } + + let mut locals = Vec::new(); + for &insn_id in state.locals() { + locals.push(jit.get_opnd(insn_id)?); + } + + let target = Target::SideExit { + pc: state.pc, + stack, + locals, + }; + Some(target) +} + impl Assembler { /// Make a C call while marking the start and end positions of it fn ccall_with_branch(&mut self, fptr: *const u8, opnds: Vec<Opnd>, branch: &Rc<Branch>) -> Opnd { @@ -744,8 +764,9 @@ impl Assembler { move |code_ptr, _| { start_branch.start_addr.set(Some(code_ptr)); }, - move |code_ptr, _| { + move |code_ptr, cb| { end_branch.end_addr.set(Some(code_ptr)); + ZJITState::add_iseq_return_addr(code_ptr.raw_ptr(cb)); }, ) } |