diff options
Diffstat (limited to 'lib/ruby_vm/rjit/insn_compiler.rb')
| -rw-r--r-- | lib/ruby_vm/rjit/insn_compiler.rb | 6046 |
1 files changed, 0 insertions, 6046 deletions
diff --git a/lib/ruby_vm/rjit/insn_compiler.rb b/lib/ruby_vm/rjit/insn_compiler.rb deleted file mode 100644 index a33ba9f468..0000000000 --- a/lib/ruby_vm/rjit/insn_compiler.rb +++ /dev/null @@ -1,6046 +0,0 @@ -# frozen_string_literal: true -module RubyVM::RJIT - class InsnCompiler - # struct rb_calling_info. Storing flags instead of ci. - CallingInfo = Struct.new(:argc, :flags, :kwarg, :ci_addr, :send_shift, :block_handler) do - def kw_splat = flags & C::VM_CALL_KW_SPLAT != 0 - end - - # @param ocb [CodeBlock] - # @param exit_compiler [RubyVM::RJIT::ExitCompiler] - def initialize(cb, ocb, exit_compiler) - @ocb = ocb - @exit_compiler = exit_compiler - - @cfunc_codegen_table = {} - register_cfunc_codegen_funcs - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - # @param insn `RubyVM::RJIT::Instruction` - def compile(jit, ctx, asm, insn) - asm.incr_counter(:rjit_insns_count) - - stack = ctx.stack_size.times.map do |stack_idx| - ctx.get_opnd_type(StackOpnd[ctx.stack_size - stack_idx - 1]).type - end - locals = jit.iseq.body.local_table_size.times.map do |local_idx| - (ctx.local_types[local_idx] || Type::Unknown).type - end - - insn_idx = format('%04d', (jit.pc.to_i - jit.iseq.body.iseq_encoded.to_i) / C.VALUE.size) - asm.comment("Insn: #{insn_idx} #{insn.name} (stack: [#{stack.join(', ')}], locals: [#{locals.join(', ')}])") - - # 83/102 - case insn.name - when :nop then nop(jit, ctx, asm) - when :getlocal then getlocal(jit, ctx, asm) - when :setlocal then setlocal(jit, ctx, asm) - when :getblockparam then getblockparam(jit, ctx, asm) - # setblockparam - when :getblockparamproxy then getblockparamproxy(jit, ctx, asm) - when :getspecial then getspecial(jit, ctx, asm) - # setspecial - when :getinstancevariable then getinstancevariable(jit, ctx, asm) - when :setinstancevariable then setinstancevariable(jit, ctx, asm) - when :getclassvariable then getclassvariable(jit, ctx, asm) - when :setclassvariable then setclassvariable(jit, ctx, asm) - when :opt_getconstant_path then opt_getconstant_path(jit, ctx, asm) - when :getconstant then getconstant(jit, ctx, asm) - # setconstant - when :getglobal then getglobal(jit, ctx, asm) - # setglobal - when :putnil then putnil(jit, ctx, asm) - when :putself then putself(jit, ctx, asm) - when :putobject then putobject(jit, ctx, asm) - when :putspecialobject then putspecialobject(jit, ctx, asm) - when :putstring then putstring(jit, ctx, asm) - when :putchilledstring then putchilledstring(jit, ctx, asm) - when :concatstrings then concatstrings(jit, ctx, asm) - when :anytostring then anytostring(jit, ctx, asm) - when :toregexp then toregexp(jit, ctx, asm) - when :intern then intern(jit, ctx, asm) - when :newarray then newarray(jit, ctx, asm) - when :duparray then duparray(jit, ctx, asm) - # duphash - when :expandarray then expandarray(jit, ctx, asm) - when :concatarray then concatarray(jit, ctx, asm) - when :splatarray then splatarray(jit, ctx, asm) - when :newhash then newhash(jit, ctx, asm) - when :newrange then newrange(jit, ctx, asm) - when :pop then pop(jit, ctx, asm) - when :dup then dup(jit, ctx, asm) - when :dupn then dupn(jit, ctx, asm) - when :swap then swap(jit, ctx, asm) - # opt_reverse - when :topn then topn(jit, ctx, asm) - when :setn then setn(jit, ctx, asm) - when :adjuststack then adjuststack(jit, ctx, asm) - when :defined then defined(jit, ctx, asm) - when :definedivar then definedivar(jit, ctx, asm) - # checkmatch - when :checkkeyword then checkkeyword(jit, ctx, asm) - # checktype - # defineclass - # definemethod - # definesmethod - when :send then send(jit, ctx, asm) - when :opt_send_without_block then opt_send_without_block(jit, ctx, asm) - when :objtostring then objtostring(jit, ctx, asm) - when :opt_str_freeze then opt_str_freeze(jit, ctx, asm) - when :opt_ary_freeze then opt_ary_freeze(jit, ctx, asm) - when :opt_hash_freeze then opt_hash_freeze(jit, ctx, asm) - when :opt_nil_p then opt_nil_p(jit, ctx, asm) - # opt_str_uminus - when :opt_newarray_send then opt_newarray_send(jit, ctx, asm) - when :invokesuper then invokesuper(jit, ctx, asm) - when :invokeblock then invokeblock(jit, ctx, asm) - when :leave then leave(jit, ctx, asm) - when :throw then throw(jit, ctx, asm) - when :jump then jump(jit, ctx, asm) - when :branchif then branchif(jit, ctx, asm) - when :branchunless then branchunless(jit, ctx, asm) - when :branchnil then branchnil(jit, ctx, asm) - # once - when :opt_case_dispatch then opt_case_dispatch(jit, ctx, asm) - when :opt_plus then opt_plus(jit, ctx, asm) - when :opt_minus then opt_minus(jit, ctx, asm) - when :opt_mult then opt_mult(jit, ctx, asm) - when :opt_div then opt_div(jit, ctx, asm) - when :opt_mod then opt_mod(jit, ctx, asm) - when :opt_eq then opt_eq(jit, ctx, asm) - when :opt_neq then opt_neq(jit, ctx, asm) - when :opt_lt then opt_lt(jit, ctx, asm) - when :opt_le then opt_le(jit, ctx, asm) - when :opt_gt then opt_gt(jit, ctx, asm) - when :opt_ge then opt_ge(jit, ctx, asm) - when :opt_ltlt then opt_ltlt(jit, ctx, asm) - when :opt_and then opt_and(jit, ctx, asm) - when :opt_or then opt_or(jit, ctx, asm) - when :opt_aref then opt_aref(jit, ctx, asm) - when :opt_aset then opt_aset(jit, ctx, asm) - # opt_aset_with - # opt_aref_with - when :opt_length then opt_length(jit, ctx, asm) - when :opt_size then opt_size(jit, ctx, asm) - when :opt_empty_p then opt_empty_p(jit, ctx, asm) - when :opt_succ then opt_succ(jit, ctx, asm) - when :opt_not then opt_not(jit, ctx, asm) - when :opt_regexpmatch2 then opt_regexpmatch2(jit, ctx, asm) - # invokebuiltin - when :opt_invokebuiltin_delegate then opt_invokebuiltin_delegate(jit, ctx, asm) - when :opt_invokebuiltin_delegate_leave then opt_invokebuiltin_delegate_leave(jit, ctx, asm) - when :getlocal_WC_0 then getlocal_WC_0(jit, ctx, asm) - when :getlocal_WC_1 then getlocal_WC_1(jit, ctx, asm) - when :setlocal_WC_0 then setlocal_WC_0(jit, ctx, asm) - when :setlocal_WC_1 then setlocal_WC_1(jit, ctx, asm) - when :putobject_INT2FIX_0_ then putobject_INT2FIX_0_(jit, ctx, asm) - when :putobject_INT2FIX_1_ then putobject_INT2FIX_1_(jit, ctx, asm) - else CantCompile - end - end - - private - - # - # Insns - # - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def nop(jit, ctx, asm) - # Do nothing - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def getlocal(jit, ctx, asm) - idx = jit.operand(0) - level = jit.operand(1) - jit_getlocal_generic(jit, ctx, asm, idx:, level:) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def getlocal_WC_0(jit, ctx, asm) - idx = jit.operand(0) - jit_getlocal_generic(jit, ctx, asm, idx:, level: 0) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def getlocal_WC_1(jit, ctx, asm) - idx = jit.operand(0) - jit_getlocal_generic(jit, ctx, asm, idx:, level: 1) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def setlocal(jit, ctx, asm) - idx = jit.operand(0) - level = jit.operand(1) - jit_setlocal_generic(jit, ctx, asm, idx:, level:) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def setlocal_WC_0(jit, ctx, asm) - idx = jit.operand(0) - jit_setlocal_generic(jit, ctx, asm, idx:, level: 0) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def setlocal_WC_1(jit, ctx, asm) - idx = jit.operand(0) - jit_setlocal_generic(jit, ctx, asm, idx:, level: 1) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def getblockparam(jit, ctx, asm) - # EP level - level = jit.operand(1) - - # Save the PC and SP because we might allocate - jit_prepare_routine_call(jit, ctx, asm) - - # A mirror of the interpreter code. Checking for the case - # where it's pushing rb_block_param_proxy. - side_exit = side_exit(jit, ctx) - - # Load environment pointer EP from CFP - ep_reg = :rax - jit_get_ep(asm, level, reg: ep_reg) - - # Bail when VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) is non zero - # FIXME: This is testing bits in the same place that the WB check is testing. - # We should combine these at some point - asm.test([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], C::VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) - - # If the frame flag has been modified, then the actual proc value is - # already in the EP and we should just use the value. - frame_flag_modified = asm.new_label('frame_flag_modified') - asm.jnz(frame_flag_modified) - - # This instruction writes the block handler to the EP. If we need to - # fire a write barrier for the write, then exit (we'll let the - # interpreter handle it so it can fire the write barrier). - # flags & VM_ENV_FLAG_WB_REQUIRED - asm.test([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], C::VM_ENV_FLAG_WB_REQUIRED) - - # if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0 - asm.jnz(side_exit) - - # Convert the block handler in to a proc - # call rb_vm_bh_to_procval(const rb_execution_context_t *ec, VALUE block_handler) - asm.mov(C_ARGS[0], EC) - # The block handler for the current frame - # note, VM_ASSERT(VM_ENV_LOCAL_P(ep)) - asm.mov(C_ARGS[1], [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) - asm.call(C.rb_vm_bh_to_procval) - - # Load environment pointer EP from CFP (again) - ep_reg = :rcx - jit_get_ep(asm, level, reg: ep_reg) - - # Write the value at the environment pointer - idx = jit.operand(0) - offs = -(C.VALUE.size * idx) - asm.mov([ep_reg, offs], C_RET); - - # Set the frame modified flag - asm.mov(:rax, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS]) # flag_check - asm.or(:rax, C::VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) # modified_flag - asm.mov([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], :rax) - - asm.write_label(frame_flag_modified) - - # Push the proc on the stack - stack_ret = ctx.stack_push(Type::Unknown) - ep_reg = :rax - jit_get_ep(asm, level, reg: ep_reg) - asm.mov(:rax, [ep_reg, offs]) - asm.mov(stack_ret, :rax) - - KeepCompiling - end - - # setblockparam - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def getblockparamproxy(jit, ctx, asm) - # To get block_handler - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - starting_context = ctx.dup # make a copy for use with jit_chain_guard - - # A mirror of the interpreter code. Checking for the case - # where it's pushing rb_block_param_proxy. - side_exit = side_exit(jit, ctx) - - # EP level - level = jit.operand(1) - - # Peek at the block handler so we can check whether it's nil - comptime_handler = jit.peek_at_block_handler(level) - - # When a block handler is present, it should always be a GC-guarded - # pointer (VM_BH_ISEQ_BLOCK_P) - if comptime_handler != 0 && comptime_handler & 0x3 != 0x1 - asm.incr_counter(:getblockpp_not_gc_guarded) - return CantCompile - end - - # Load environment pointer EP from CFP - ep_reg = :rax - jit_get_ep(asm, level, reg: ep_reg) - - # Bail when VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) is non zero - asm.test([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], C::VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) - asm.jnz(counted_exit(side_exit, :getblockpp_block_param_modified)) - - # Load the block handler for the current frame - # note, VM_ASSERT(VM_ENV_LOCAL_P(ep)) - block_handler = :rax - asm.mov(block_handler, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) - - # Specialize compilation for the case where no block handler is present - if comptime_handler == 0 - # Bail if there is a block handler - asm.cmp(block_handler, 0) - - jit_chain_guard(:jnz, jit, starting_context, asm, counted_exit(side_exit, :getblockpp_block_handler_none)) - - putobject(jit, ctx, asm, val: Qnil) - else - # Block handler is a tagged pointer. Look at the tag. 0x03 is from VM_BH_ISEQ_BLOCK_P(). - asm.and(block_handler, 0x3) - - # Bail unless VM_BH_ISEQ_BLOCK_P(bh). This also checks for null. - asm.cmp(block_handler, 0x1) - - jit_chain_guard(:jnz, jit, starting_context, asm, counted_exit(side_exit, :getblockpp_not_iseq_block)) - - # Push rb_block_param_proxy. It's a root, so no need to use jit_mov_gc_ptr. - top = ctx.stack_push(Type::BlockParamProxy) - asm.mov(:rax, C.rb_block_param_proxy) - asm.mov(top, :rax) - end - - jump_to_next_insn(jit, ctx, asm) - - EndBlock - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def getspecial(jit, ctx, asm) - # This takes two arguments, key and type - # key is only used when type == 0 - # A non-zero type determines which type of backref to fetch - #rb_num_t key = jit.jit_get_arg(0); - rtype = jit.operand(1) - - if rtype == 0 - # not yet implemented - return CantCompile; - elsif rtype & 0x01 != 0 - # Fetch a "special" backref based on a char encoded by shifting by 1 - - # Can raise if matchdata uninitialized - jit_prepare_routine_call(jit, ctx, asm) - - # call rb_backref_get() - asm.comment('rb_backref_get') - asm.call(C.rb_backref_get) - - asm.mov(C_ARGS[0], C_RET) # backref - case [rtype >> 1].pack('c') - in ?& - asm.comment("rb_reg_last_match") - asm.call(C.rb_reg_last_match) - in ?` - asm.comment("rb_reg_match_pre") - asm.call(C.rb_reg_match_pre) - in ?' - asm.comment("rb_reg_match_post") - asm.call(C.rb_reg_match_post) - in ?+ - asm.comment("rb_reg_match_last") - asm.call(C.rb_reg_match_last) - end - - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - - KeepCompiling - else - # Fetch the N-th match from the last backref based on type shifted by 1 - - # Can raise if matchdata uninitialized - jit_prepare_routine_call(jit, ctx, asm) - - # call rb_backref_get() - asm.comment('rb_backref_get') - asm.call(C.rb_backref_get) - - # rb_reg_nth_match((int)(type >> 1), backref); - asm.comment('rb_reg_nth_match') - asm.mov(C_ARGS[0], rtype >> 1) - asm.mov(C_ARGS[1], C_RET) # backref - asm.call(C.rb_reg_nth_match) - - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - end - - # setspecial - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def getinstancevariable(jit, ctx, asm) - # Specialize on a compile-time receiver, and split a block for chain guards - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - id = jit.operand(0) - comptime_obj = jit.peek_at_self - - jit_getivar(jit, ctx, asm, comptime_obj, id, nil, SelfOpnd) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def setinstancevariable(jit, ctx, asm) - starting_context = ctx.dup # make a copy for use with jit_chain_guard - - # Defer compilation so we can specialize on a runtime `self` - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - ivar_name = jit.operand(0) - comptime_receiver = jit.peek_at_self - - # If the comptime receiver is frozen, writing an IV will raise an exception - # and we don't want to JIT code to deal with that situation. - if C.rb_obj_frozen_p(comptime_receiver) - asm.incr_counter(:setivar_frozen) - return CantCompile - end - - # Check if the comptime receiver is a T_OBJECT - receiver_t_object = C::BUILTIN_TYPE(comptime_receiver) == C::T_OBJECT - - # If the receiver isn't a T_OBJECT, or uses a custom allocator, - # then just write out the IV write as a function call. - # too-complex shapes can't use index access, so we use rb_ivar_get for them too. - if !receiver_t_object || shape_too_complex?(comptime_receiver) || ctx.chain_depth >= 10 - asm.comment('call rb_vm_setinstancevariable') - - ic = jit.operand(1) - - # The function could raise exceptions. - # Note that this modifies REG_SP, which is why we do it first - jit_prepare_routine_call(jit, ctx, asm) - - # Get the operands from the stack - val_opnd = ctx.stack_pop(1) - - # Call rb_vm_setinstancevariable(iseq, obj, id, val, ic); - asm.mov(:rdi, jit.iseq.to_i) - asm.mov(:rsi, [CFP, C.rb_control_frame_t.offsetof(:self)]) - asm.mov(:rdx, ivar_name) - asm.mov(:rcx, val_opnd) - asm.mov(:r8, ic) - asm.call(C.rb_vm_setinstancevariable) - else - # Get the iv index - shape_id = C.rb_shape_get_shape_id(comptime_receiver) - ivar_index = C.rb_shape_get_iv_index(shape_id, ivar_name) - - # Get the receiver - asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) - - # Generate a side exit - side_exit = side_exit(jit, ctx) - - # Upgrade type - guard_object_is_heap(jit, ctx, asm, :rax, SelfOpnd, :setivar_not_heap) - - asm.comment('guard shape') - asm.cmp(DwordPtr[:rax, C.rb_shape_id_offset], shape_id) - megamorphic_side_exit = counted_exit(side_exit, :setivar_megamorphic) - jit_chain_guard(:jne, jit, starting_context, asm, megamorphic_side_exit) - - # If we don't have an instance variable index, then we need to - # transition out of the current shape. - if ivar_index.nil? - shape = C.rb_shape_get_shape_by_id(shape_id) - - current_capacity = shape.capacity - dest_shape = C.rb_shape_get_next_no_warnings(shape, comptime_receiver, ivar_name) - new_shape_id = C.rb_shape_id(dest_shape) - - if new_shape_id == C::OBJ_TOO_COMPLEX_SHAPE_ID - asm.incr_counter(:setivar_too_complex) - return CantCompile - end - - ivar_index = shape.next_iv_index - - # If the new shape has a different capacity, we need to - # reallocate the object. - needs_extension = dest_shape.capacity != shape.capacity - - if needs_extension - # Generate the C call so that runtime code will increase - # the capacity and set the buffer. - asm.mov(C_ARGS[0], :rax) - asm.mov(C_ARGS[1], current_capacity) - asm.mov(C_ARGS[2], dest_shape.capacity) - asm.call(C.rb_ensure_iv_list_size) - - # Load the receiver again after the function call - asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) - end - - write_val = ctx.stack_pop(1) - jit_write_iv(asm, comptime_receiver, :rax, :rcx, ivar_index, write_val, needs_extension) - - # Store the new shape - asm.comment('write shape') - asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) # reload after jit_write_iv - asm.mov(DwordPtr[:rax, C.rb_shape_id_offset], new_shape_id) - else - # If the iv index already exists, then we don't need to - # transition to a new shape. The reason is because we find - # the iv index by searching up the shape tree. If we've - # made the transition already, then there's no reason to - # update the shape on the object. Just set the IV. - write_val = ctx.stack_pop(1) - jit_write_iv(asm, comptime_receiver, :rax, :rcx, ivar_index, write_val, false) - end - - skip_wb = asm.new_label('skip_wb') - # If the value we're writing is an immediate, we don't need to WB - asm.test(write_val, C::RUBY_IMMEDIATE_MASK) - asm.jnz(skip_wb) - - # If the value we're writing is nil or false, we don't need to WB - asm.cmp(write_val, Qnil) - asm.jbe(skip_wb) - - asm.comment('write barrier') - asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:self)]) # reload after jit_write_iv - asm.mov(C_ARGS[1], write_val) - asm.call(C.rb_gc_writebarrier) - - asm.write_label(skip_wb) - end - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def getclassvariable(jit, ctx, asm) - # rb_vm_getclassvariable can raise exceptions. - jit_prepare_routine_call(jit, ctx, asm) - - asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:iseq)]) - asm.mov(C_ARGS[1], CFP) - asm.mov(C_ARGS[2], jit.operand(0)) - asm.mov(C_ARGS[3], jit.operand(1)) - asm.call(C.rb_vm_getclassvariable) - - top = ctx.stack_push(Type::Unknown) - asm.mov(top, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def setclassvariable(jit, ctx, asm) - # rb_vm_setclassvariable can raise exceptions. - jit_prepare_routine_call(jit, ctx, asm) - - asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:iseq)]) - asm.mov(C_ARGS[1], CFP) - asm.mov(C_ARGS[2], jit.operand(0)) - asm.mov(C_ARGS[3], ctx.stack_pop(1)) - asm.mov(C_ARGS[4], jit.operand(1)) - asm.call(C.rb_vm_setclassvariable) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_getconstant_path(jit, ctx, asm) - # Cut the block for invalidation - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - ic = C.iseq_inline_constant_cache.new(jit.operand(0)) - idlist = ic.segments - - # Make sure there is an exit for this block as the interpreter might want - # to invalidate this block from rb_rjit_constant_ic_update(). - # For now, we always take an entry exit even if it was a side exit. - Invariants.ensure_block_entry_exit(jit, cause: 'opt_getconstant_path') - - # See vm_ic_hit_p(). The same conditions are checked in yjit_constant_ic_update(). - ice = ic.entry - if ice.nil? - # In this case, leave a block that unconditionally side exits - # for the interpreter to invalidate. - asm.incr_counter(:optgetconst_not_cached) - return CantCompile - end - - if ice.ic_cref # with cref - # Cache is keyed on a certain lexical scope. Use the interpreter's cache. - side_exit = side_exit(jit, ctx) - - # Call function to verify the cache. It doesn't allocate or call methods. - asm.mov(C_ARGS[0], ic.to_i) - asm.mov(C_ARGS[1], [CFP, C.rb_control_frame_t.offsetof(:ep)]) - asm.call(C.rb_vm_ic_hit_p) - - # Check the result. SysV only specifies one byte for _Bool return values, - # so it's important we only check one bit to ignore the higher bits in the register. - asm.test(C_RET, 1) - asm.jz(counted_exit(side_exit, :optgetconst_cache_miss)) - - asm.mov(:rax, ic.to_i) # inline_cache - asm.mov(:rax, [:rax, C.iseq_inline_constant_cache.offsetof(:entry)]) # ic_entry - asm.mov(:rax, [:rax, C.iseq_inline_constant_cache_entry.offsetof(:value)]) # ic_entry_val - - # Push ic->entry->value - stack_top = ctx.stack_push(Type::Unknown) - asm.mov(stack_top, :rax) - else # without cref - # TODO: implement this - # Optimize for single ractor mode. - # if !assume_single_ractor_mode(jit, ocb) - # return CantCompile - # end - - # Invalidate output code on any constant writes associated with - # constants referenced within the current block. - Invariants.assume_stable_constant_names(jit, idlist) - - putobject(jit, ctx, asm, val: ice.value) - end - - jump_to_next_insn(jit, ctx, asm) - EndBlock - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def getconstant(jit, ctx, asm) - id = jit.operand(0) - - # vm_get_ev_const can raise exceptions. - jit_prepare_routine_call(jit, ctx, asm) - - allow_nil_opnd = ctx.stack_pop(1) - klass_opnd = ctx.stack_pop(1) - - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], klass_opnd) - asm.mov(C_ARGS[2], id) - asm.mov(C_ARGS[3], allow_nil_opnd) - asm.call(C.rb_vm_get_ev_const) - - top = ctx.stack_push(Type::Unknown) - asm.mov(top, C_RET) - - KeepCompiling - end - - # setconstant - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def getglobal(jit, ctx, asm) - gid = jit.operand(0) - - # Save the PC and SP because we might make a Ruby call for warning - jit_prepare_routine_call(jit, ctx, asm) - - asm.mov(C_ARGS[0], gid) - asm.call(C.rb_gvar_get) - - top = ctx.stack_push(Type::Unknown) - asm.mov(top, C_RET) - - KeepCompiling - end - - # setglobal - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def putnil(jit, ctx, asm) - putobject(jit, ctx, asm, val: Qnil) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def putself(jit, ctx, asm) - stack_top = ctx.stack_push_self - asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) - asm.mov(stack_top, :rax) - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def putobject(jit, ctx, asm, val: jit.operand(0)) - # Push it to the stack - val_type = Type.from(C.to_ruby(val)) - stack_top = ctx.stack_push(val_type) - if asm.imm32?(val) - asm.mov(stack_top, val) - else # 64-bit immediates can't be directly written to memory - asm.mov(:rax, val) - asm.mov(stack_top, :rax) - end - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def putspecialobject(jit, ctx, asm) - object_type = jit.operand(0) - if object_type == C::VM_SPECIAL_OBJECT_VMCORE - stack_top = ctx.stack_push(Type::UnknownHeap) - asm.mov(:rax, C.rb_mRubyVMFrozenCore) - asm.mov(stack_top, :rax) - KeepCompiling - else - # TODO: implement for VM_SPECIAL_OBJECT_CBASE and - # VM_SPECIAL_OBJECT_CONST_BASE - CantCompile - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def putstring(jit, ctx, asm) - put_val = jit.operand(0, ruby: true) - - # Save the PC and SP because the callee will allocate - jit_prepare_routine_call(jit, ctx, asm) - - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], to_value(put_val)) - asm.mov(C_ARGS[2], 0) - asm.call(C.rb_ec_str_resurrect) - - stack_top = ctx.stack_push(Type::TString) - asm.mov(stack_top, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def putchilledstring(jit, ctx, asm) - put_val = jit.operand(0, ruby: true) - - # Save the PC and SP because the callee will allocate - jit_prepare_routine_call(jit, ctx, asm) - - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], to_value(put_val)) - asm.mov(C_ARGS[2], 1) - asm.call(C.rb_ec_str_resurrect) - - stack_top = ctx.stack_push(Type::TString) - asm.mov(stack_top, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def concatstrings(jit, ctx, asm) - n = jit.operand(0) - - # Save the PC and SP because we are allocating - jit_prepare_routine_call(jit, ctx, asm) - - asm.lea(:rax, ctx.sp_opnd(-C.VALUE.size * n)) - - # call rb_str_concat_literals(size_t n, const VALUE *strings); - asm.mov(C_ARGS[0], n) - asm.mov(C_ARGS[1], :rax) - asm.call(C.rb_str_concat_literals) - - ctx.stack_pop(n) - stack_ret = ctx.stack_push(Type::TString) - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def anytostring(jit, ctx, asm) - # Save the PC and SP since we might call #to_s - jit_prepare_routine_call(jit, ctx, asm) - - str = ctx.stack_pop(1) - val = ctx.stack_pop(1) - - asm.mov(C_ARGS[0], str) - asm.mov(C_ARGS[1], val) - asm.call(C.rb_obj_as_string_result) - - # Push the return value - stack_ret = ctx.stack_push(Type::TString) - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def toregexp(jit, ctx, asm) - opt = jit.operand(0, signed: true) - cnt = jit.operand(1) - - # Save the PC and SP because this allocates an object and could - # raise an exception. - jit_prepare_routine_call(jit, ctx, asm) - - asm.lea(:rax, ctx.sp_opnd(-C.VALUE.size * cnt)) # values_ptr - ctx.stack_pop(cnt) - - asm.mov(C_ARGS[0], 0) - asm.mov(C_ARGS[1], cnt) - asm.mov(C_ARGS[2], :rax) # values_ptr - asm.call(C.rb_ary_tmp_new_from_values) - - # Save the array so we can clear it later - asm.push(C_RET) - asm.push(C_RET) # Alignment - - asm.mov(C_ARGS[0], C_RET) - asm.mov(C_ARGS[1], opt) - asm.call(C.rb_reg_new_ary) - - # The actual regex is in RAX now. Pop the temp array from - # rb_ary_tmp_new_from_values into C arg regs so we can clear it - asm.pop(:rcx) # Alignment - asm.pop(:rcx) # ary - - # The value we want to push on the stack is in RAX right now - stack_ret = ctx.stack_push(Type::UnknownHeap) - asm.mov(stack_ret, C_RET) - - # Clear the temp array. - asm.mov(C_ARGS[0], :rcx) # ary - asm.call(C.rb_ary_clear) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def intern(jit, ctx, asm) - # Save the PC and SP because we might allocate - jit_prepare_routine_call(jit, ctx, asm); - - str = ctx.stack_pop(1) - asm.mov(C_ARGS[0], str) - asm.call(C.rb_str_intern) - - # Push the return value - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def newarray(jit, ctx, asm) - n = jit.operand(0) - - # Save the PC and SP because we are allocating - jit_prepare_routine_call(jit, ctx, asm) - - # If n is 0, then elts is never going to be read, so we can just pass null - if n == 0 - values_ptr = 0 - else - asm.comment('load pointer to array elts') - offset_magnitude = C.VALUE.size * n - values_opnd = ctx.sp_opnd(-(offset_magnitude)) - asm.lea(:rax, values_opnd) - values_ptr = :rax - end - - # call rb_ec_ary_new_from_values(struct rb_execution_context_struct *ec, long n, const VALUE *elts); - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], n) - asm.mov(C_ARGS[2], values_ptr) - asm.call(C.rb_ec_ary_new_from_values) - - ctx.stack_pop(n) - stack_ret = ctx.stack_push(Type::TArray) - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def duparray(jit, ctx, asm) - ary = jit.operand(0) - - # Save the PC and SP because we are allocating - jit_prepare_routine_call(jit, ctx, asm) - - # call rb_ary_resurrect(VALUE ary); - asm.comment('call rb_ary_resurrect') - asm.mov(C_ARGS[0], ary) - asm.call(C.rb_ary_resurrect) - - stack_ret = ctx.stack_push(Type::TArray) - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # duphash - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def expandarray(jit, ctx, asm) - # Both arguments are rb_num_t which is unsigned - num = jit.operand(0) - flag = jit.operand(1) - - # If this instruction has the splat flag, then bail out. - if flag & 0x01 != 0 - asm.incr_counter(:expandarray_splat) - return CantCompile - end - - # If this instruction has the postarg flag, then bail out. - if flag & 0x02 != 0 - asm.incr_counter(:expandarray_postarg) - return CantCompile - end - - side_exit = side_exit(jit, ctx) - - array_opnd = ctx.stack_opnd(0) - array_stack_opnd = StackOpnd[0] - - # num is the number of requested values. If there aren't enough in the - # array then we're going to push on nils. - if ctx.get_opnd_type(array_stack_opnd) == Type::Nil - ctx.stack_pop(1) # pop after using the type info - # special case for a, b = nil pattern - # push N nils onto the stack - num.times do - push_opnd = ctx.stack_push(Type::Nil) - asm.mov(push_opnd, Qnil) - end - return KeepCompiling - end - - # Move the array from the stack and check that it's an array. - asm.mov(:rax, array_opnd) - guard_object_is_array(jit, ctx, asm, :rax, :rcx, array_stack_opnd, :expandarray_not_array) - ctx.stack_pop(1) # pop after using the type info - - # If we don't actually want any values, then just return. - if num == 0 - return KeepCompiling - end - - jit_array_len(asm, :rax, :rcx) - - # Only handle the case where the number of values in the array is greater - # than or equal to the number of values requested. - asm.cmp(:rcx, num) - asm.jl(counted_exit(side_exit, :expandarray_rhs_too_small)) - - # Conditionally load the address of the heap array into REG1. - # (struct RArray *)(obj)->as.heap.ptr - #asm.mov(:rax, array_opnd) - asm.mov(:rcx, [:rax, C.RBasic.offsetof(:flags)]) - asm.test(:rcx, C::RARRAY_EMBED_FLAG); - asm.mov(:rcx, [:rax, C.RArray.offsetof(:as, :heap, :ptr)]) - - # Load the address of the embedded array into REG1. - # (struct RArray *)(obj)->as.ary - asm.lea(:rax, [:rax, C.RArray.offsetof(:as, :ary)]) - - asm.cmovnz(:rcx, :rax) - - # Loop backward through the array and push each element onto the stack. - (num - 1).downto(0).each do |i| - top = ctx.stack_push(Type::Unknown) - asm.mov(:rax, [:rcx, i * C.VALUE.size]) - asm.mov(top, :rax) - end - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def concatarray(jit, ctx, asm) - # Save the PC and SP because the callee may allocate - # Note that this modifies REG_SP, which is why we do it first - jit_prepare_routine_call(jit, ctx, asm) - - # Get the operands from the stack - ary2st_opnd = ctx.stack_pop(1) - ary1_opnd = ctx.stack_pop(1) - - # Call rb_vm_concat_array(ary1, ary2st) - asm.mov(C_ARGS[0], ary1_opnd) - asm.mov(C_ARGS[1], ary2st_opnd) - asm.call(C.rb_vm_concat_array) - - stack_ret = ctx.stack_push(Type::TArray) - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def splatarray(jit, ctx, asm) - flag = jit.operand(0) - - # Save the PC and SP because the callee may allocate - # Note that this modifies REG_SP, which is why we do it first - jit_prepare_routine_call(jit, ctx, asm) - - # Get the operands from the stack - ary_opnd = ctx.stack_pop(1) - - # Call rb_vm_splat_array(flag, ary) - asm.mov(C_ARGS[0], flag) - asm.mov(C_ARGS[1], ary_opnd) - asm.call(C.rb_vm_splat_array) - - stack_ret = ctx.stack_push(Type::TArray) - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def newhash(jit, ctx, asm) - num = jit.operand(0) - - # Save the PC and SP because we are allocating - jit_prepare_routine_call(jit, ctx, asm) - - if num != 0 - # val = rb_hash_new_with_size(num / 2); - asm.mov(C_ARGS[0], num / 2) - asm.call(C.rb_hash_new_with_size) - - # Save the allocated hash as we want to push it after insertion - asm.push(C_RET) - asm.push(C_RET) # x86 alignment - - # Get a pointer to the values to insert into the hash - asm.lea(:rcx, ctx.stack_opnd(num - 1)) - - # rb_hash_bulk_insert(num, STACK_ADDR_FROM_TOP(num), val); - asm.mov(C_ARGS[0], num) - asm.mov(C_ARGS[1], :rcx) - asm.mov(C_ARGS[2], C_RET) - asm.call(C.rb_hash_bulk_insert) - - asm.pop(:rax) - asm.pop(:rax) - - ctx.stack_pop(num) - stack_ret = ctx.stack_push(Type::Hash) - asm.mov(stack_ret, :rax) - else - # val = rb_hash_new(); - asm.call(C.rb_hash_new) - stack_ret = ctx.stack_push(Type::Hash) - asm.mov(stack_ret, C_RET) - end - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def newrange(jit, ctx, asm) - flag = jit.operand(0) - - # rb_range_new() allocates and can raise - jit_prepare_routine_call(jit, ctx, asm) - - # val = rb_range_new(low, high, (int)flag); - asm.mov(C_ARGS[0], ctx.stack_opnd(1)) - asm.mov(C_ARGS[1], ctx.stack_opnd(0)) - asm.mov(C_ARGS[2], flag) - asm.call(C.rb_range_new) - - ctx.stack_pop(2) - stack_ret = ctx.stack_push(Type::UnknownHeap) - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def pop(jit, ctx, asm) - ctx.stack_pop - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def dup(jit, ctx, asm) - dup_val = ctx.stack_opnd(0) - mapping, tmp_type = ctx.get_opnd_mapping(StackOpnd[0]) - - loc0 = ctx.stack_push_mapping([mapping, tmp_type]) - asm.mov(:rax, dup_val) - asm.mov(loc0, :rax) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def dupn(jit, ctx, asm) - n = jit.operand(0) - - # In practice, seems to be only used for n==2 - if n != 2 - return CantCompile - end - - opnd1 = ctx.stack_opnd(1) - opnd0 = ctx.stack_opnd(0) - - mapping1 = ctx.get_opnd_mapping(StackOpnd[1]) - mapping0 = ctx.get_opnd_mapping(StackOpnd[0]) - - dst1 = ctx.stack_push_mapping(mapping1) - asm.mov(:rax, opnd1) - asm.mov(dst1, :rax) - - dst0 = ctx.stack_push_mapping(mapping0) - asm.mov(:rax, opnd0) - asm.mov(dst0, :rax) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def swap(jit, ctx, asm) - stack_swap(jit, ctx, asm, 0, 1) - KeepCompiling - end - - # opt_reverse - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def topn(jit, ctx, asm) - n = jit.operand(0) - - top_n_val = ctx.stack_opnd(n) - mapping = ctx.get_opnd_mapping(StackOpnd[n]) - loc0 = ctx.stack_push_mapping(mapping) - asm.mov(:rax, top_n_val) - asm.mov(loc0, :rax) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def setn(jit, ctx, asm) - n = jit.operand(0) - - top_val = ctx.stack_pop(0) - dst_opnd = ctx.stack_opnd(n) - asm.mov(:rax, top_val) - asm.mov(dst_opnd, :rax) - - mapping = ctx.get_opnd_mapping(StackOpnd[0]) - ctx.set_opnd_mapping(StackOpnd[n], mapping) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def adjuststack(jit, ctx, asm) - n = jit.operand(0) - ctx.stack_pop(n) - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def defined(jit, ctx, asm) - op_type = jit.operand(0) - obj = jit.operand(1, ruby: true) - pushval = jit.operand(2, ruby: true) - - # Save the PC and SP because the callee may allocate - # Note that this modifies REG_SP, which is why we do it first - jit_prepare_routine_call(jit, ctx, asm) - - # Get the operands from the stack - v_opnd = ctx.stack_pop(1) - - # Call vm_defined(ec, reg_cfp, op_type, obj, v) - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], CFP) - asm.mov(C_ARGS[2], op_type) - asm.mov(C_ARGS[3], to_value(obj)) - asm.mov(C_ARGS[4], v_opnd) - asm.call(C.rb_vm_defined) - - asm.test(C_RET, 255) - asm.mov(:rax, Qnil) - asm.mov(:rcx, to_value(pushval)) - asm.cmovnz(:rax, :rcx) - - # Push the return value onto the stack - out_type = if C::SPECIAL_CONST_P(pushval) - Type::UnknownImm - else - Type::Unknown - end - stack_ret = ctx.stack_push(out_type) - asm.mov(stack_ret, :rax) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def definedivar(jit, ctx, asm) - # Defer compilation so we can specialize base on a runtime receiver - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - ivar_name = jit.operand(0) - # Value that will be pushed on the stack if the ivar is defined. In practice this is always the - # string "instance-variable". If the ivar is not defined, nil will be pushed instead. - pushval = jit.operand(2, ruby: true) - - # Get the receiver - recv = :rcx - asm.mov(recv, [CFP, C.rb_control_frame_t.offsetof(:self)]) - - # Specialize base on compile time values - comptime_receiver = jit.peek_at_self - - if shape_too_complex?(comptime_receiver) - # Fall back to calling rb_ivar_defined - - # Save the PC and SP because the callee may allocate - # Note that this modifies REG_SP, which is why we do it first - jit_prepare_routine_call(jit, ctx, asm) # clobbers :rax - - # Call rb_ivar_defined(recv, ivar_name) - asm.mov(C_ARGS[0], recv) - asm.mov(C_ARGS[1], ivar_name) - asm.call(C.rb_ivar_defined) - - # if (rb_ivar_defined(recv, ivar_name)) { - # val = pushval; - # } - asm.test(C_RET, 255) - asm.mov(:rax, Qnil) - asm.mov(:rcx, to_value(pushval)) - asm.cmovnz(:rax, :rcx) - - # Push the return value onto the stack - out_type = C::SPECIAL_CONST_P(pushval) ? Type::UnknownImm : Type::Unknown - stack_ret = ctx.stack_push(out_type) - asm.mov(stack_ret, :rax) - - return KeepCompiling - end - - shape_id = C.rb_shape_get_shape_id(comptime_receiver) - ivar_exists = C.rb_shape_get_iv_index(shape_id, ivar_name) - - side_exit = side_exit(jit, ctx) - - # Guard heap object (recv_opnd must be used before stack_pop) - guard_object_is_heap(jit, ctx, asm, recv, SelfOpnd) - - shape_opnd = DwordPtr[recv, C.rb_shape_id_offset] - - asm.comment('guard shape') - asm.cmp(shape_opnd, shape_id) - jit_chain_guard(:jne, jit, ctx, asm, side_exit) - - result = ivar_exists ? C.to_value(pushval) : Qnil - putobject(jit, ctx, asm, val: result) - - # Jump to next instruction. This allows guard chains to share the same successor. - jump_to_next_insn(jit, ctx, asm) - - return EndBlock - end - - # checkmatch - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def checkkeyword(jit, ctx, asm) - # When a keyword is unspecified past index 32, a hash will be used - # instead. This can only happen in iseqs taking more than 32 keywords. - if jit.iseq.body.param.keyword.num >= 32 - return CantCompile - end - - # The EP offset to the undefined bits local - bits_offset = jit.operand(0) - - # The index of the keyword we want to check - index = jit.operand(1, signed: true) - - # Load environment pointer EP - ep_reg = :rax - jit_get_ep(asm, 0, reg: ep_reg) - - # VALUE kw_bits = *(ep - bits) - bits_opnd = [ep_reg, C.VALUE.size * -bits_offset] - - # unsigned int b = (unsigned int)FIX2ULONG(kw_bits); - # if ((b & (0x01 << idx))) { - # - # We can skip the FIX2ULONG conversion by shifting the bit we test - bit_test = 0x01 << (index + 1) - asm.test(bits_opnd, bit_test) - asm.mov(:rax, Qfalse) - asm.mov(:rcx, Qtrue) - asm.cmovz(:rax, :rcx) - - stack_ret = ctx.stack_push(Type::UnknownImm) - asm.mov(stack_ret, :rax) - - KeepCompiling - end - - # checktype - # defineclass - # definemethod - # definesmethod - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def send(jit, ctx, asm) - # Specialize on a compile-time receiver, and split a block for chain guards - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - cd = C.rb_call_data.new(jit.operand(0)) - blockiseq = jit.operand(1) - - # calling->ci - mid = C.vm_ci_mid(cd.ci) - calling = build_calling(ci: cd.ci, block_handler: blockiseq) - - if calling.flags & C::VM_CALL_FORWARDING != 0 - return CantCompile - end - - # vm_sendish - cme, comptime_recv_klass = jit_search_method(jit, ctx, asm, mid, calling) - if cme == CantCompile - return CantCompile - end - jit_call_general(jit, ctx, asm, mid, calling, cme, comptime_recv_klass) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_send_without_block(jit, ctx, asm, cd: C.rb_call_data.new(jit.operand(0))) - # Specialize on a compile-time receiver, and split a block for chain guards - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - # calling->ci - mid = C.vm_ci_mid(cd.ci) - calling = build_calling(ci: cd.ci, block_handler: C::VM_BLOCK_HANDLER_NONE) - - # vm_sendish - cme, comptime_recv_klass = jit_search_method(jit, ctx, asm, mid, calling) - if cme == CantCompile - return CantCompile - end - jit_call_general(jit, ctx, asm, mid, calling, cme, comptime_recv_klass) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def objtostring(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - recv = ctx.stack_opnd(0) - comptime_recv = jit.peek_at_stack(0) - - if C.RB_TYPE_P(comptime_recv, C::RUBY_T_STRING) - side_exit = side_exit(jit, ctx) - - jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv, StackOpnd[0], comptime_recv, side_exit) - # No work needed. The string value is already on the top of the stack. - KeepCompiling - else - cd = C.rb_call_data.new(jit.operand(0)) - opt_send_without_block(jit, ctx, asm, cd:) - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_ary_freeze(jit, ctx, asm) - unless Invariants.assume_bop_not_redefined(jit, C::ARRAY_REDEFINED_OP_FLAG, C::BOP_FREEZE) - return CantCompile; - end - - ary = jit.operand(0, ruby: true) - - # Push the return value onto the stack - stack_ret = ctx.stack_push(Type::CArray) - asm.mov(:rax, to_value(ary)) - asm.mov(stack_ret, :rax) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_hash_freeze(jit, ctx, asm) - unless Invariants.assume_bop_not_redefined(jit, C::HASH_REDEFINED_OP_FLAG, C::BOP_FREEZE) - return CantCompile; - end - - hash = jit.operand(0, ruby: true) - - # Push the return value onto the stack - stack_ret = ctx.stack_push(Type::CHash) - asm.mov(:rax, to_value(hash)) - asm.mov(stack_ret, :rax) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_str_freeze(jit, ctx, asm) - unless Invariants.assume_bop_not_redefined(jit, C::STRING_REDEFINED_OP_FLAG, C::BOP_FREEZE) - return CantCompile; - end - - str = jit.operand(0, ruby: true) - - # Push the return value onto the stack - stack_ret = ctx.stack_push(Type::CString) - asm.mov(:rax, to_value(str)) - asm.mov(stack_ret, :rax) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_nil_p(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # opt_str_uminus - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_newarray_send(jit, ctx, asm) - type = C.ID2SYM jit.operand(1) - - case type - when :min then opt_newarray_min(jit, ctx, asm) - when :max then opt_newarray_max(jit, ctx, asm) - when :hash then opt_newarray_hash(jit, ctx, asm) - else - return CantCompile - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_newarray_min(jit, ctx, asm) - num = jit.operand(0) - - # Save the PC and SP because we may allocate - jit_prepare_routine_call(jit, ctx, asm) - - offset_magnitude = C.VALUE.size * num - values_opnd = ctx.sp_opnd(-offset_magnitude) - asm.lea(:rax, values_opnd) - - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], num) - asm.mov(C_ARGS[2], :rax) - asm.call(C.rb_vm_opt_newarray_min) - - ctx.stack_pop(num) - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_newarray_max(jit, ctx, asm) - num = jit.operand(0) - - # Save the PC and SP because we may allocate - jit_prepare_routine_call(jit, ctx, asm) - - offset_magnitude = C.VALUE.size * num - values_opnd = ctx.sp_opnd(-offset_magnitude) - asm.lea(:rax, values_opnd) - - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], num) - asm.mov(C_ARGS[2], :rax) - asm.call(C.rb_vm_opt_newarray_max) - - ctx.stack_pop(num) - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_newarray_hash(jit, ctx, asm) - num = jit.operand(0) - - # Save the PC and SP because we may allocate - jit_prepare_routine_call(jit, ctx, asm) - - offset_magnitude = C.VALUE.size * num - values_opnd = ctx.sp_opnd(-offset_magnitude) - asm.lea(:rax, values_opnd) - - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], num) - asm.mov(C_ARGS[2], :rax) - asm.call(C.rb_vm_opt_newarray_hash) - - ctx.stack_pop(num) - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def invokesuper(jit, ctx, asm) - cd = C.rb_call_data.new(jit.operand(0)) - block = jit.operand(1) - - # Defer compilation so we can specialize on class of receiver - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - me = C.rb_vm_frame_method_entry(jit.cfp) - if me.nil? - return CantCompile - end - - # FIXME: We should track and invalidate this block when this cme is invalidated - current_defined_class = me.defined_class - mid = me.def.original_id - - if me.to_i != C.rb_callable_method_entry(current_defined_class, me.called_id).to_i - # Though we likely could generate this call, as we are only concerned - # with the method entry remaining valid, assume_method_lookup_stable - # below requires that the method lookup matches as well - return CantCompile - end - - # vm_search_normal_superclass - rbasic_klass = C.to_ruby(C.RBasic.new(C.to_value(current_defined_class)).klass) - if C::BUILTIN_TYPE(current_defined_class) == C::RUBY_T_ICLASS && C::BUILTIN_TYPE(rbasic_klass) == C::RUBY_T_MODULE && \ - C::FL_TEST_RAW(rbasic_klass, C::RMODULE_IS_REFINEMENT) - return CantCompile - end - comptime_superclass = C.rb_class_get_superclass(C.RCLASS_ORIGIN(current_defined_class)) - - ci = cd.ci - argc = C.vm_ci_argc(ci) - - ci_flags = C.vm_ci_flag(ci) - - # Don't JIT calls that aren't simple - # Note, not using VM_CALL_ARGS_SIMPLE because sometimes we pass a block. - - if ci_flags & C::VM_CALL_KWARG != 0 - asm.incr_counter(:send_keywords) - return CantCompile - end - if ci_flags & C::VM_CALL_KW_SPLAT != 0 - asm.incr_counter(:send_kw_splat) - return CantCompile - end - if ci_flags & C::VM_CALL_ARGS_BLOCKARG != 0 - asm.incr_counter(:send_block_arg) - return CantCompile - end - - # Ensure we haven't rebound this method onto an incompatible class. - # In the interpreter we try to avoid making this check by performing some - # cheaper calculations first, but since we specialize on the method entry - # and so only have to do this once at compile time this is fine to always - # check and side exit. - comptime_recv = jit.peek_at_stack(argc) - unless C.obj_is_kind_of(comptime_recv, current_defined_class) - return CantCompile - end - - # Do method lookup - cme = C.rb_callable_method_entry(comptime_superclass, mid) - - if cme.nil? - return CantCompile - end - - # Check that we'll be able to write this method dispatch before generating checks - cme_def_type = cme.def.type - if cme_def_type != C::VM_METHOD_TYPE_ISEQ && cme_def_type != C::VM_METHOD_TYPE_CFUNC - # others unimplemented - return CantCompile - end - - asm.comment('guard known me') - lep_opnd = :rax - jit_get_lep(jit, asm, reg: lep_opnd) - ep_me_opnd = [lep_opnd, C.VALUE.size * C::VM_ENV_DATA_INDEX_ME_CREF] - - asm.mov(:rcx, me.to_i) - asm.cmp(ep_me_opnd, :rcx) - asm.jne(counted_exit(side_exit(jit, ctx), :invokesuper_me_changed)) - - if block == C::VM_BLOCK_HANDLER_NONE - # Guard no block passed - # rb_vm_frame_block_handler(GET_EC()->cfp) == VM_BLOCK_HANDLER_NONE - # note, we assume VM_ASSERT(VM_ENV_LOCAL_P(ep)) - # - # TODO: this could properly forward the current block handler, but - # would require changes to gen_send_* - asm.comment('guard no block given') - ep_specval_opnd = [lep_opnd, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL] - asm.cmp(ep_specval_opnd, C::VM_BLOCK_HANDLER_NONE) - asm.jne(counted_exit(side_exit(jit, ctx), :invokesuper_block)) - end - - # We need to assume that both our current method entry and the super - # method entry we invoke remain stable - Invariants.assume_method_lookup_stable(jit, me) - Invariants.assume_method_lookup_stable(jit, cme) - - # Method calls may corrupt types - ctx.clear_local_types - - calling = build_calling(ci:, block_handler: block) - case cme_def_type - in C::VM_METHOD_TYPE_ISEQ - iseq = def_iseq_ptr(cme.def) - frame_type = C::VM_FRAME_MAGIC_METHOD | C::VM_ENV_FLAG_LOCAL - jit_call_iseq(jit, ctx, asm, cme, calling, iseq, frame_type:) - in C::VM_METHOD_TYPE_CFUNC - jit_call_cfunc(jit, ctx, asm, cme, calling) - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def invokeblock(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - # Get call info - cd = C.rb_call_data.new(jit.operand(0)) - calling = build_calling(ci: cd.ci, block_handler: :captured) - - # Get block_handler - cfp = jit.cfp - lep = C.rb_vm_ep_local_ep(cfp.ep) - comptime_handler = lep[C::VM_ENV_DATA_INDEX_SPECVAL] - - # Handle each block_handler type - if comptime_handler == C::VM_BLOCK_HANDLER_NONE # no block given - asm.incr_counter(:invokeblock_none) - CantCompile - elsif comptime_handler & 0x3 == 0x1 # VM_BH_ISEQ_BLOCK_P - asm.comment('get local EP') - ep_reg = :rax - jit_get_lep(jit, asm, reg: ep_reg) - asm.mov(:rax, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler_opnd - - asm.comment('guard block_handler type') - side_exit = side_exit(jit, ctx) - asm.mov(:rcx, :rax) - asm.and(:rcx, 0x3) # block_handler is a tagged pointer - asm.cmp(:rcx, 0x1) # VM_BH_ISEQ_BLOCK_P - tag_changed_exit = counted_exit(side_exit, :invokeblock_tag_changed) - jit_chain_guard(:jne, jit, ctx, asm, tag_changed_exit) - - comptime_captured = C.rb_captured_block.new(comptime_handler & ~0x3) - comptime_iseq = comptime_captured.code.iseq - - asm.comment('guard known ISEQ') - asm.and(:rax, ~0x3) # captured - asm.mov(:rax, [:rax, C.VALUE.size * 2]) # captured->iseq - asm.mov(:rcx, comptime_iseq.to_i) - asm.cmp(:rax, :rcx) - block_changed_exit = counted_exit(side_exit, :invokeblock_iseq_block_changed) - jit_chain_guard(:jne, jit, ctx, asm, block_changed_exit) - - jit_call_iseq(jit, ctx, asm, nil, calling, comptime_iseq, frame_type: C::VM_FRAME_MAGIC_BLOCK) - elsif comptime_handler & 0x3 == 0x3 # VM_BH_IFUNC_P - # We aren't handling CALLER_SETUP_ARG and CALLER_REMOVE_EMPTY_KW_SPLAT yet. - if calling.flags & C::VM_CALL_ARGS_SPLAT != 0 - asm.incr_counter(:invokeblock_ifunc_args_splat) - return CantCompile - end - if calling.flags & C::VM_CALL_KW_SPLAT != 0 - asm.incr_counter(:invokeblock_ifunc_kw_splat) - return CantCompile - end - - asm.comment('get local EP') - jit_get_lep(jit, asm, reg: :rax) - asm.mov(:rcx, [:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler_opnd - - asm.comment('guard block_handler type'); - side_exit = side_exit(jit, ctx) - asm.mov(:rax, :rcx) # block_handler_opnd - asm.and(:rax, 0x3) # tag_opnd: block_handler is a tagged pointer - asm.cmp(:rax, 0x3) # VM_BH_IFUNC_P - tag_changed_exit = counted_exit(side_exit, :invokeblock_tag_changed) - jit_chain_guard(:jne, jit, ctx, asm, tag_changed_exit) - - # The cfunc may not be leaf - jit_prepare_routine_call(jit, ctx, asm) # clobbers :rax - - asm.comment('call ifunc') - asm.and(:rcx, ~0x3) # captured_opnd - asm.lea(:rax, ctx.sp_opnd(-calling.argc * C.VALUE.size)) # argv - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], :rcx) # captured_opnd - asm.mov(C_ARGS[2], calling.argc) - asm.mov(C_ARGS[3], :rax) # argv - asm.call(C.rb_vm_yield_with_cfunc) - - ctx.stack_pop(calling.argc) - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - - # cfunc calls may corrupt types - ctx.clear_local_types - - # Share the successor with other chains - jump_to_next_insn(jit, ctx, asm) - EndBlock - elsif symbol?(comptime_handler) - asm.incr_counter(:invokeblock_symbol) - CantCompile - else # Proc - asm.incr_counter(:invokeblock_proc) - CantCompile - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def leave(jit, ctx, asm) - assert_equal(ctx.stack_size, 1) - - jit_check_ints(jit, ctx, asm) - - asm.comment('pop stack frame') - asm.lea(:rax, [CFP, C.rb_control_frame_t.size]) - asm.mov(CFP, :rax) - asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], :rax) - - # Return a value (for compile_leave_exit) - ret_opnd = ctx.stack_pop - asm.mov(:rax, ret_opnd) - - # Set caller's SP and push a value to its stack (for JIT) - asm.mov(SP, [CFP, C.rb_control_frame_t.offsetof(:sp)]) # Note: SP is in the position after popping a receiver and arguments - asm.mov([SP], :rax) - - # Jump to cfp->jit_return - asm.jmp([CFP, -C.rb_control_frame_t.size + C.rb_control_frame_t.offsetof(:jit_return)]) - - EndBlock - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def throw(jit, ctx, asm) - throw_state = jit.operand(0) - asm.mov(:rcx, ctx.stack_pop(1)) # throwobj - - # THROW_DATA_NEW allocates. Save SP for GC and PC for allocation tracing as - # well as handling the catch table. However, not using jit_prepare_routine_call - # since we don't need a patch point for this implementation. - jit_save_pc(jit, asm) # clobbers rax - jit_save_sp(ctx, asm) - - # rb_vm_throw verifies it's a valid throw, sets ec->tag->state, and returns throw - # data, which is throwobj or a vm_throw_data wrapping it. When ec->tag->state is - # set, JIT code callers will handle the throw with vm_exec_handle_exception. - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], CFP) - asm.mov(C_ARGS[2], throw_state) - # asm.mov(C_ARGS[3], :rcx) # same reg - asm.call(C.rb_vm_throw) - - asm.comment('exit from throw') - asm.pop(SP) - asm.pop(EC) - asm.pop(CFP) - - # return C_RET as C_RET - asm.ret - EndBlock - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jump(jit, ctx, asm) - # Check for interrupts, but only on backward branches that may create loops - jump_offset = jit.operand(0, signed: true) - if jump_offset < 0 - jit_check_ints(jit, ctx, asm) - end - - pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset) - jit_direct_jump(jit.iseq, pc, ctx, asm) - EndBlock - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def branchif(jit, ctx, asm) - # Check for interrupts, but only on backward branches that may create loops - jump_offset = jit.operand(0, signed: true) - if jump_offset < 0 - jit_check_ints(jit, ctx, asm) - end - - # Get the branch target instruction offsets - next_pc = jit.pc + C.VALUE.size * jit.insn.len - jump_pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset) - - val_type = ctx.get_opnd_type(StackOpnd[0]) - val_opnd = ctx.stack_pop(1) - - if (result = val_type.known_truthy) != nil - target_pc = result ? jump_pc : next_pc - jit_direct_jump(jit.iseq, target_pc, ctx, asm) - else - # This `test` sets ZF only for Qnil and Qfalse, which let jz jump. - asm.test(val_opnd, ~Qnil) - - # Set stubs - branch_stub = BranchStub.new( - iseq: jit.iseq, - shape: Default, - target0: BranchTarget.new(ctx:, pc: jump_pc), # branch target - target1: BranchTarget.new(ctx:, pc: next_pc), # fallthrough - ) - branch_stub.target0.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true) - @ocb.write(ocb_asm) - end - branch_stub.target1.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false) - @ocb.write(ocb_asm) - end - - # Jump to target0 on jnz - branch_stub.compile = compile_branchif(branch_stub) - branch_stub.compile.call(asm) - end - - EndBlock - end - - def compile_branchif(branch_stub) # Proc escapes arguments in memory - proc do |branch_asm| - branch_asm.comment("branchif #{branch_stub.shape}") - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.jnz(branch_stub.target0.address) - branch_asm.jmp(branch_stub.target1.address) - in Next0 - branch_asm.jz(branch_stub.target1.address) - in Next1 - branch_asm.jnz(branch_stub.target0.address) - end - end - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def branchunless(jit, ctx, asm) - # Check for interrupts, but only on backward branches that may create loops - jump_offset = jit.operand(0, signed: true) - if jump_offset < 0 - jit_check_ints(jit, ctx, asm) - end - - # Get the branch target instruction offsets - next_pc = jit.pc + C.VALUE.size * jit.insn.len - jump_pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset) - - val_type = ctx.get_opnd_type(StackOpnd[0]) - val_opnd = ctx.stack_pop(1) - - if (result = val_type.known_truthy) != nil - target_pc = result ? next_pc : jump_pc - jit_direct_jump(jit.iseq, target_pc, ctx, asm) - else - # This `test` sets ZF only for Qnil and Qfalse, which let jz jump. - asm.test(val_opnd, ~Qnil) - - # Set stubs - branch_stub = BranchStub.new( - iseq: jit.iseq, - shape: Default, - target0: BranchTarget.new(ctx:, pc: jump_pc), # branch target - target1: BranchTarget.new(ctx:, pc: next_pc), # fallthrough - ) - branch_stub.target0.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true) - @ocb.write(ocb_asm) - end - branch_stub.target1.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false) - @ocb.write(ocb_asm) - end - - # Jump to target0 on jz - branch_stub.compile = compile_branchunless(branch_stub) - branch_stub.compile.call(asm) - end - - EndBlock - end - - def compile_branchunless(branch_stub) # Proc escapes arguments in memory - proc do |branch_asm| - branch_asm.comment("branchunless #{branch_stub.shape}") - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.jz(branch_stub.target0.address) - branch_asm.jmp(branch_stub.target1.address) - in Next0 - branch_asm.jnz(branch_stub.target1.address) - in Next1 - branch_asm.jz(branch_stub.target0.address) - end - end - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def branchnil(jit, ctx, asm) - # Check for interrupts, but only on backward branches that may create loops - jump_offset = jit.operand(0, signed: true) - if jump_offset < 0 - jit_check_ints(jit, ctx, asm) - end - - # Get the branch target instruction offsets - next_pc = jit.pc + C.VALUE.size * jit.insn.len - jump_pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset) - - val_type = ctx.get_opnd_type(StackOpnd[0]) - val_opnd = ctx.stack_pop(1) - - if (result = val_type.known_nil) != nil - target_pc = result ? jump_pc : next_pc - jit_direct_jump(jit.iseq, target_pc, ctx, asm) - else - asm.cmp(val_opnd, Qnil) - - # Set stubs - branch_stub = BranchStub.new( - iseq: jit.iseq, - shape: Default, - target0: BranchTarget.new(ctx:, pc: jump_pc), # branch target - target1: BranchTarget.new(ctx:, pc: next_pc), # fallthrough - ) - branch_stub.target0.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true) - @ocb.write(ocb_asm) - end - branch_stub.target1.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false) - @ocb.write(ocb_asm) - end - - # Jump to target0 on je - branch_stub.compile = compile_branchnil(branch_stub) - branch_stub.compile.call(asm) - end - - EndBlock - end - - def compile_branchnil(branch_stub) # Proc escapes arguments in memory - proc do |branch_asm| - branch_asm.comment("branchnil #{branch_stub.shape}") - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.je(branch_stub.target0.address) - branch_asm.jmp(branch_stub.target1.address) - in Next0 - branch_asm.jne(branch_stub.target1.address) - in Next1 - branch_asm.je(branch_stub.target0.address) - end - end - end - end - - # once - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_case_dispatch(jit, ctx, asm) - # Normally this instruction would lookup the key in a hash and jump to an - # offset based on that. - # Instead we can take the fallback case and continue with the next - # instruction. - # We'd hope that our jitted code will be sufficiently fast without the - # hash lookup, at least for small hashes, but it's worth revisiting this - # assumption in the future. - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - starting_context = ctx.dup - - case_hash = jit.operand(0, ruby: true) - else_offset = jit.operand(1) - - # Try to reorder case/else branches so that ones that are actually used come first. - # Supporting only Fixnum for now so that the implementation can be an equality check. - key_opnd = ctx.stack_pop(1) - comptime_key = jit.peek_at_stack(0) - - # Check that all cases are fixnums to avoid having to register BOP assumptions on - # all the types that case hashes support. This spends compile time to save memory. - if fixnum?(comptime_key) && comptime_key <= 2**32 && C.rb_hash_keys(case_hash).all? { |key| fixnum?(key) } - unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_EQQ) - return CantCompile - end - - # Check if the key is the same value - asm.cmp(key_opnd, to_value(comptime_key)) - side_exit = side_exit(jit, starting_context) - jit_chain_guard(:jne, jit, starting_context, asm, side_exit) - - # Get the offset for the compile-time key - offset = C.rb_hash_stlike_lookup(case_hash, comptime_key) - # NOTE: If we hit the else branch with various values, it could negatively impact the performance. - jump_offset = offset || else_offset - - # Jump to the offset of case or else - target_pc = jit.pc + (jit.insn.len + jump_offset) * C.VALUE.size - jit_direct_jump(jit.iseq, target_pc, ctx, asm) - EndBlock - else - KeepCompiling # continue with === branches - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_plus(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - comptime_recv = jit.peek_at_stack(1) - comptime_obj = jit.peek_at_stack(0) - - if fixnum?(comptime_recv) && fixnum?(comptime_obj) - unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_PLUS) - return CantCompile - end - - # Check that both operands are fixnums - guard_two_fixnums(jit, ctx, asm) - - obj_opnd = ctx.stack_pop - recv_opnd = ctx.stack_pop - - asm.mov(:rax, recv_opnd) - asm.sub(:rax, 1) # untag - asm.mov(:rcx, obj_opnd) - asm.add(:rax, :rcx) - asm.jo(side_exit(jit, ctx)) - - dst_opnd = ctx.stack_push(Type::Fixnum) - asm.mov(dst_opnd, :rax) - - KeepCompiling - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_minus(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - comptime_recv = jit.peek_at_stack(1) - comptime_obj = jit.peek_at_stack(0) - - if fixnum?(comptime_recv) && fixnum?(comptime_obj) - unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_MINUS) - return CantCompile - end - - # Check that both operands are fixnums - guard_two_fixnums(jit, ctx, asm) - - obj_opnd = ctx.stack_pop - recv_opnd = ctx.stack_pop - - asm.mov(:rax, recv_opnd) - asm.mov(:rcx, obj_opnd) - asm.sub(:rax, :rcx) - asm.jo(side_exit(jit, ctx)) - asm.add(:rax, 1) # re-tag - - dst_opnd = ctx.stack_push(Type::Fixnum) - asm.mov(dst_opnd, :rax) - - KeepCompiling - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_mult(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_div(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_mod(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - if two_fixnums_on_stack?(jit) - unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_MOD) - return CantCompile - end - - # Check that both operands are fixnums - guard_two_fixnums(jit, ctx, asm) - - # Get the operands and destination from the stack - arg1 = ctx.stack_pop(1) - arg0 = ctx.stack_pop(1) - - # Check for arg0 % 0 - asm.cmp(arg1, 0) - asm.je(side_exit(jit, ctx)) - - # Call rb_fix_mod_fix(VALUE recv, VALUE obj) - asm.mov(C_ARGS[0], arg0) - asm.mov(C_ARGS[1], arg1) - asm.call(C.rb_fix_mod_fix) - - # Push the return value onto the stack - stack_ret = ctx.stack_push(Type::Fixnum) - asm.mov(stack_ret, C_RET) - - KeepCompiling - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_eq(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - if jit_equality_specialized(jit, ctx, asm, true) - jump_to_next_insn(jit, ctx, asm) - EndBlock - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_neq(jit, ctx, asm) - # opt_neq is passed two rb_call_data as arguments: - # first for ==, second for != - neq_cd = C.rb_call_data.new(jit.operand(1)) - opt_send_without_block(jit, ctx, asm, cd: neq_cd) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_lt(jit, ctx, asm) - jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovl, bop: C::BOP_LT) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_le(jit, ctx, asm) - jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovle, bop: C::BOP_LE) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_gt(jit, ctx, asm) - jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovg, bop: C::BOP_GT) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_ge(jit, ctx, asm) - jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovge, bop: C::BOP_GE) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_ltlt(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_and(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - if two_fixnums_on_stack?(jit) - unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_AND) - return CantCompile - end - - # Check that both operands are fixnums - guard_two_fixnums(jit, ctx, asm) - - # Get the operands and destination from the stack - arg1 = ctx.stack_pop(1) - arg0 = ctx.stack_pop(1) - - asm.comment('bitwise and') - asm.mov(:rax, arg0) - asm.and(:rax, arg1) - - # Push the return value onto the stack - dst = ctx.stack_push(Type::Fixnum) - asm.mov(dst, :rax) - - KeepCompiling - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_or(jit, ctx, asm) - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - if two_fixnums_on_stack?(jit) - unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_OR) - return CantCompile - end - - # Check that both operands are fixnums - guard_two_fixnums(jit, ctx, asm) - - # Get the operands and destination from the stack - asm.comment('bitwise or') - arg1 = ctx.stack_pop(1) - arg0 = ctx.stack_pop(1) - - # Do the bitwise or arg0 | arg1 - asm.mov(:rax, arg0) - asm.or(:rax, arg1) - - # Push the return value onto the stack - dst = ctx.stack_push(Type::Fixnum) - asm.mov(dst, :rax) - - KeepCompiling - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_aref(jit, ctx, asm) - cd = C.rb_call_data.new(jit.operand(0)) - argc = C.vm_ci_argc(cd.ci) - - if argc != 1 - asm.incr_counter(:optaref_argc_not_one) - return CantCompile - end - - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - comptime_recv = jit.peek_at_stack(1) - comptime_obj = jit.peek_at_stack(0) - - side_exit = side_exit(jit, ctx) - - if C.rb_class_of(comptime_recv) == Array && fixnum?(comptime_obj) - unless Invariants.assume_bop_not_redefined(jit, C::ARRAY_REDEFINED_OP_FLAG, C::BOP_AREF) - return CantCompile - end - - idx_opnd = ctx.stack_opnd(0) - recv_opnd = ctx.stack_opnd(1) - - not_array_exit = counted_exit(side_exit, :optaref_recv_not_array) - jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv_opnd, StackOpnd[1], comptime_recv, not_array_exit) - - # Bail if idx is not a FIXNUM - asm.mov(:rax, idx_opnd) - asm.test(:rax, C::RUBY_FIXNUM_FLAG) - asm.jz(counted_exit(side_exit, :optaref_arg_not_fixnum)) - - # Call VALUE rb_ary_entry_internal(VALUE ary, long offset). - # It never raises or allocates, so we don't need to write to cfp->pc. - asm.sar(:rax, 1) # Convert fixnum to int - asm.mov(C_ARGS[0], recv_opnd) - asm.mov(C_ARGS[1], :rax) - asm.call(C.rb_ary_entry_internal) - - # Pop the argument and the receiver - ctx.stack_pop(2) - - # Push the return value onto the stack - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - - # Let guard chains share the same successor - jump_to_next_insn(jit, ctx, asm) - EndBlock - elsif C.rb_class_of(comptime_recv) == Hash - unless Invariants.assume_bop_not_redefined(jit, C::HASH_REDEFINED_OP_FLAG, C::BOP_AREF) - return CantCompile - end - - recv_opnd = ctx.stack_opnd(1) - - # Guard that the receiver is a Hash - not_hash_exit = counted_exit(side_exit, :optaref_recv_not_hash) - jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv_opnd, StackOpnd[1], comptime_recv, not_hash_exit) - - # Prepare to call rb_hash_aref(). It might call #hash on the key. - jit_prepare_routine_call(jit, ctx, asm) - - asm.comment('call rb_hash_aref') - key_opnd = ctx.stack_opnd(0) - recv_opnd = ctx.stack_opnd(1) - asm.mov(:rdi, recv_opnd) - asm.mov(:rsi, key_opnd) - asm.call(C.rb_hash_aref) - - # Pop the key and the receiver - ctx.stack_pop(2) - - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - - # Let guard chains share the same successor - jump_to_next_insn(jit, ctx, asm) - EndBlock - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_aset(jit, ctx, asm) - # Defer compilation so we can specialize on a runtime `self` - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - comptime_recv = jit.peek_at_stack(2) - comptime_key = jit.peek_at_stack(1) - - # Get the operands from the stack - recv = ctx.stack_opnd(2) - key = ctx.stack_opnd(1) - _val = ctx.stack_opnd(0) - - if C.rb_class_of(comptime_recv) == Array && fixnum?(comptime_key) - side_exit = side_exit(jit, ctx) - - # Guard receiver is an Array - jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv, StackOpnd[2], comptime_recv, side_exit) - - # Guard key is a fixnum - jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_key), key, StackOpnd[1], comptime_key, side_exit) - - # We might allocate or raise - jit_prepare_routine_call(jit, ctx, asm) - - asm.comment('call rb_ary_store') - recv = ctx.stack_opnd(2) - key = ctx.stack_opnd(1) - val = ctx.stack_opnd(0) - asm.mov(:rax, key) - asm.sar(:rax, 1) # FIX2LONG(key) - asm.mov(C_ARGS[0], recv) - asm.mov(C_ARGS[1], :rax) - asm.mov(C_ARGS[2], val) - asm.call(C.rb_ary_store) - - # rb_ary_store returns void - # stored value should still be on stack - val = ctx.stack_opnd(0) - - # Push the return value onto the stack - ctx.stack_pop(3) - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(:rax, val) - asm.mov(stack_ret, :rax) - - jump_to_next_insn(jit, ctx, asm) - EndBlock - elsif C.rb_class_of(comptime_recv) == Hash - side_exit = side_exit(jit, ctx) - - # Guard receiver is a Hash - jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv, StackOpnd[2], comptime_recv, side_exit) - - # We might allocate or raise - jit_prepare_routine_call(jit, ctx, asm) - - # Call rb_hash_aset - recv = ctx.stack_opnd(2) - key = ctx.stack_opnd(1) - val = ctx.stack_opnd(0) - asm.mov(C_ARGS[0], recv) - asm.mov(C_ARGS[1], key) - asm.mov(C_ARGS[2], val) - asm.call(C.rb_hash_aset) - - # Push the return value onto the stack - ctx.stack_pop(3) - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - - jump_to_next_insn(jit, ctx, asm) - EndBlock - else - opt_send_without_block(jit, ctx, asm) - end - end - - # opt_aset_with - # opt_aref_with - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_length(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_size(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_empty_p(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_succ(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_not(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_regexpmatch2(jit, ctx, asm) - opt_send_without_block(jit, ctx, asm) - end - - # invokebuiltin - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_invokebuiltin_delegate(jit, ctx, asm) - bf = C.rb_builtin_function.new(jit.operand(0)) - bf_argc = bf.argc - start_index = jit.operand(1) - - # ec, self, and arguments - if bf_argc + 2 > C_ARGS.size - return CantCompile - end - - # If the calls don't allocate, do they need up to date PC, SP? - jit_prepare_routine_call(jit, ctx, asm) - - # Call the builtin func (ec, recv, arg1, arg2, ...) - asm.comment('call builtin func') - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], [CFP, C.rb_control_frame_t.offsetof(:self)]) - - # Copy arguments from locals - if bf_argc > 0 - # Load environment pointer EP from CFP - asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:ep)]) - - bf_argc.times do |i| - table_size = jit.iseq.body.local_table_size - offs = -table_size - C::VM_ENV_DATA_SIZE + 1 + start_index + i - asm.mov(C_ARGS[2 + i], [:rax, offs * C.VALUE.size]) - end - end - asm.call(bf.func_ptr) - - # Push the return value - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - - KeepCompiling - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def opt_invokebuiltin_delegate_leave(jit, ctx, asm) - opt_invokebuiltin_delegate(jit, ctx, asm) - # opt_invokebuiltin_delegate is always followed by leave insn - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def putobject_INT2FIX_0_(jit, ctx, asm) - putobject(jit, ctx, asm, val: C.to_value(0)) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def putobject_INT2FIX_1_(jit, ctx, asm) - putobject(jit, ctx, asm, val: C.to_value(1)) - end - - # - # C func - # - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_true(jit, ctx, asm, argc, _known_recv_class) - return false if argc != 0 - asm.comment('nil? == true') - ctx.stack_pop(1) - stack_ret = ctx.stack_push(Type::True) - asm.mov(stack_ret, Qtrue) - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_false(jit, ctx, asm, argc, _known_recv_class) - return false if argc != 0 - asm.comment('nil? == false') - ctx.stack_pop(1) - stack_ret = ctx.stack_push(Type::False) - asm.mov(stack_ret, Qfalse) - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_kernel_is_a(jit, ctx, asm, argc, known_recv_class) - if argc != 1 - return false - end - - # If this is a super call we might not know the class - if known_recv_class.nil? - return false - end - - # Important note: The output code will simply `return true/false`. - # Correctness follows from: - # - `known_recv_class` implies there is a guard scheduled before here - # for a particular `CLASS_OF(lhs)`. - # - We guard that rhs is identical to the compile-time sample - # - In general, for any two Class instances A, B, `A < B` does not change at runtime. - # Class#superclass is stable. - - sample_rhs = jit.peek_at_stack(0) - sample_lhs = jit.peek_at_stack(1) - - # We are not allowing module here because the module hierarchy can change at runtime. - if C.RB_TYPE_P(sample_rhs, C::RUBY_T_CLASS) - return false - end - sample_is_a = C.obj_is_kind_of(sample_lhs, sample_rhs) - - side_exit = side_exit(jit, ctx) - asm.comment('Kernel#is_a?') - asm.mov(:rax, to_value(sample_rhs)) - asm.cmp(ctx.stack_opnd(0), :rax) - asm.jne(counted_exit(side_exit, :send_is_a_class_mismatch)) - - ctx.stack_pop(2) - - if sample_is_a - stack_ret = ctx.stack_push(Type::True) - asm.mov(stack_ret, Qtrue) - else - stack_ret = ctx.stack_push(Type::False) - asm.mov(stack_ret, Qfalse) - end - return true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_kernel_instance_of(jit, ctx, asm, argc, known_recv_class) - if argc != 1 - return false - end - - # If this is a super call we might not know the class - if known_recv_class.nil? - return false - end - - # Important note: The output code will simply `return true/false`. - # Correctness follows from: - # - `known_recv_class` implies there is a guard scheduled before here - # for a particular `CLASS_OF(lhs)`. - # - We guard that rhs is identical to the compile-time sample - # - For a particular `CLASS_OF(lhs)`, `rb_obj_class(lhs)` does not change. - # (because for any singleton class `s`, `s.superclass.equal?(s.attached_object.class)`) - - sample_rhs = jit.peek_at_stack(0) - sample_lhs = jit.peek_at_stack(1) - - # Filters out cases where the C implementation raises - unless C.RB_TYPE_P(sample_rhs, C::RUBY_T_CLASS) || C.RB_TYPE_P(sample_rhs, C::RUBY_T_MODULE) - return false - end - - # We need to grab the class here to deal with singleton classes. - # Instance of grabs the "real class" of the object rather than the - # singleton class. - sample_lhs_real_class = C.rb_obj_class(sample_lhs) - - sample_instance_of = (sample_lhs_real_class == sample_rhs) - - side_exit = side_exit(jit, ctx) - asm.comment('Kernel#instance_of?') - asm.mov(:rax, to_value(sample_rhs)) - asm.cmp(ctx.stack_opnd(0), :rax) - asm.jne(counted_exit(side_exit, :send_instance_of_class_mismatch)) - - ctx.stack_pop(2) - - if sample_instance_of - stack_ret = ctx.stack_push(Type::True) - asm.mov(stack_ret, Qtrue) - else - stack_ret = ctx.stack_push(Type::False) - asm.mov(stack_ret, Qfalse) - end - return true; - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_obj_not(jit, ctx, asm, argc, _known_recv_class) - return false if argc != 0 - recv_type = ctx.get_opnd_type(StackOpnd[0]) - - case recv_type.known_truthy - in false - asm.comment('rb_obj_not(nil_or_false)') - ctx.stack_pop(1) - out_opnd = ctx.stack_push(Type::True) - asm.mov(out_opnd, Qtrue) - in true - # Note: recv_type != Type::Nil && recv_type != Type::False. - asm.comment('rb_obj_not(truthy)') - ctx.stack_pop(1) - out_opnd = ctx.stack_push(Type::False) - asm.mov(out_opnd, Qfalse) - in nil - asm.comment('rb_obj_not') - - recv = ctx.stack_pop - # This `test` sets ZF only for Qnil and Qfalse, which let cmovz set. - asm.test(recv, ~Qnil) - asm.mov(:rax, Qfalse) - asm.mov(:rcx, Qtrue) - asm.cmovz(:rax, :rcx) - - stack_ret = ctx.stack_push(Type::UnknownImm) - asm.mov(stack_ret, :rax) - end - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_obj_equal(jit, ctx, asm, argc, _known_recv_class) - return false if argc != 1 - asm.comment('equal?') - obj1 = ctx.stack_pop(1) - obj2 = ctx.stack_pop(1) - - asm.mov(:rax, obj1) - asm.mov(:rcx, obj2) - asm.cmp(:rax, :rcx) - asm.mov(:rax, Qfalse) - asm.mov(:rcx, Qtrue) - asm.cmove(:rax, :rcx) - - stack_ret = ctx.stack_push(Type::UnknownImm) - asm.mov(stack_ret, :rax) - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_obj_not_equal(jit, ctx, asm, argc, _known_recv_class) - return false if argc != 1 - jit_equality_specialized(jit, ctx, asm, false) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_mod_eqq(jit, ctx, asm, argc, _known_recv_class) - return false if argc != 1 - - asm.comment('Module#===') - # By being here, we know that the receiver is a T_MODULE or a T_CLASS, because Module#=== can - # only live on these objects. With that, we can call rb_obj_is_kind_of() without - # jit_prepare_routine_call() or a control frame push because it can't raise, allocate, or call - # Ruby methods with these inputs. - # Note the difference in approach from Kernel#is_a? because we don't get a free guard for the - # right hand side. - lhs = ctx.stack_opnd(1) # the module - rhs = ctx.stack_opnd(0) - asm.mov(C_ARGS[0], rhs); - asm.mov(C_ARGS[1], lhs); - asm.call(C.rb_obj_is_kind_of) - - # Return the result - ctx.stack_pop(2) - stack_ret = ctx.stack_push(Type::UnknownImm) - asm.mov(stack_ret, C_RET) - - return true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_int_equal(jit, ctx, asm, argc, _known_recv_class) - return false if argc != 1 - return false unless two_fixnums_on_stack?(jit) - - guard_two_fixnums(jit, ctx, asm) - - # Compare the arguments - asm.comment('rb_int_equal') - arg1 = ctx.stack_pop(1) - arg0 = ctx.stack_pop(1) - asm.mov(:rax, arg1) - asm.cmp(arg0, :rax) - asm.mov(:rax, Qfalse) - asm.mov(:rcx, Qtrue) - asm.cmove(:rax, :rcx) - - stack_ret = ctx.stack_push(Type::UnknownImm) - asm.mov(stack_ret, :rax) - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_int_mul(jit, ctx, asm, argc, _known_recv_class) - return false if argc != 1 - return false unless two_fixnums_on_stack?(jit) - - guard_two_fixnums(jit, ctx, asm) - - asm.comment('rb_int_mul') - y_opnd = ctx.stack_pop - x_opnd = ctx.stack_pop - asm.mov(C_ARGS[0], x_opnd) - asm.mov(C_ARGS[1], y_opnd) - asm.call(C.rb_fix_mul_fix) - - ret_opnd = ctx.stack_push(Type::Unknown) - asm.mov(ret_opnd, C_RET) - true - end - - def jit_rb_int_div(jit, ctx, asm, argc, _known_recv_class) - return false if argc != 1 - return false unless two_fixnums_on_stack?(jit) - - guard_two_fixnums(jit, ctx, asm) - - asm.comment('rb_int_div') - y_opnd = ctx.stack_pop - x_opnd = ctx.stack_pop - asm.mov(:rax, y_opnd) - asm.cmp(:rax, C.to_value(0)) - asm.je(side_exit(jit, ctx)) - - asm.mov(C_ARGS[0], x_opnd) - asm.mov(C_ARGS[1], :rax) - asm.call(C.rb_fix_div_fix) - - ret_opnd = ctx.stack_push(Type::Unknown) - asm.mov(ret_opnd, C_RET) - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_int_aref(jit, ctx, asm, argc, _known_recv_class) - return false if argc != 1 - return false unless two_fixnums_on_stack?(jit) - - guard_two_fixnums(jit, ctx, asm) - - asm.comment('rb_int_aref') - y_opnd = ctx.stack_pop - x_opnd = ctx.stack_pop - - asm.mov(C_ARGS[0], x_opnd) - asm.mov(C_ARGS[1], y_opnd) - asm.call(C.rb_fix_aref) - - ret_opnd = ctx.stack_push(Type::UnknownImm) - asm.mov(ret_opnd, C_RET) - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_str_empty_p(jit, ctx, asm, argc, known_recv_class) - recv_opnd = ctx.stack_pop(1) - out_opnd = ctx.stack_push(Type::UnknownImm) - - asm.comment('get string length') - asm.mov(:rax, recv_opnd) - str_len_opnd = [:rax, C.RString.offsetof(:len)] - - asm.cmp(str_len_opnd, 0) - asm.mov(:rax, Qfalse) - asm.mov(:rcx, Qtrue) - asm.cmove(:rax, :rcx) - asm.mov(out_opnd, :rax) - - return true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_str_to_s(jit, ctx, asm, argc, known_recv_class) - return false if argc != 0 - if known_recv_class == String - asm.comment('to_s on plain string') - # The method returns the receiver, which is already on the stack. - # No stack movement. - return true - end - false - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_str_bytesize(jit, ctx, asm, argc, known_recv_class) - asm.comment('String#bytesize') - - recv = ctx.stack_pop(1) - asm.mov(C_ARGS[0], recv) - asm.call(C.rb_str_bytesize) - - out_opnd = ctx.stack_push(Type::Fixnum) - asm.mov(out_opnd, C_RET) - - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_str_concat(jit, ctx, asm, argc, known_recv_class) - # The << operator can accept integer codepoints for characters - # as the argument. We only specially optimise string arguments. - # If the peeked-at compile time argument is something other than - # a string, assume it won't be a string later either. - comptime_arg = jit.peek_at_stack(0) - unless C.RB_TYPE_P(comptime_arg, C::RUBY_T_STRING) - return false - end - - # Guard that the concat argument is a string - asm.mov(:rax, ctx.stack_opnd(0)) - guard_object_is_string(jit, ctx, asm, :rax, :rcx, StackOpnd[0]) - - # Guard buffers from GC since rb_str_buf_append may allocate. During the VM lock on GC, - # other Ractors may trigger global invalidation, so we need ctx.clear_local_types. - # PC is used on errors like Encoding::CompatibilityError raised by rb_str_buf_append. - jit_prepare_routine_call(jit, ctx, asm) - - concat_arg = ctx.stack_pop(1) - recv = ctx.stack_pop(1) - - # Test if string encodings differ. If different, use rb_str_append. If the same, - # use rb_yjit_str_simple_append, which calls rb_str_cat. - asm.comment('<< on strings') - - # Take receiver's object flags XOR arg's flags. If any - # string-encoding flags are different between the two, - # the encodings don't match. - recv_reg = :rax - asm.mov(recv_reg, recv) - concat_arg_reg = :rcx - asm.mov(concat_arg_reg, concat_arg) - asm.mov(recv_reg, [recv_reg, C.RBasic.offsetof(:flags)]) - asm.mov(concat_arg_reg, [concat_arg_reg, C.RBasic.offsetof(:flags)]) - asm.xor(recv_reg, concat_arg_reg) - asm.test(recv_reg, C::RUBY_ENCODING_MASK) - - # Push once, use the resulting operand in both branches below. - stack_ret = ctx.stack_push(Type::TString) - - enc_mismatch = asm.new_label('enc_mismatch') - asm.jnz(enc_mismatch) - - # If encodings match, call the simple append function and jump to return - asm.mov(C_ARGS[0], recv) - asm.mov(C_ARGS[1], concat_arg) - asm.call(C.rjit_str_simple_append) - ret_label = asm.new_label('func_return') - asm.mov(stack_ret, C_RET) - asm.jmp(ret_label) - - # If encodings are different, use a slower encoding-aware concatenate - asm.write_label(enc_mismatch) - asm.mov(C_ARGS[0], recv) - asm.mov(C_ARGS[1], concat_arg) - asm.call(C.rb_str_buf_append) - asm.mov(stack_ret, C_RET) - # Drop through to return - - asm.write_label(ret_label) - - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_str_uplus(jit, ctx, asm, argc, _known_recv_class) - if argc != 0 - return false - end - - # We allocate when we dup the string - jit_prepare_routine_call(jit, ctx, asm) - - asm.comment('Unary plus on string') - asm.mov(:rax, ctx.stack_pop(1)) # recv_opnd - asm.mov(:rcx, [:rax, C.RBasic.offsetof(:flags)]) # flags_opnd - asm.test(:rcx, C::RUBY_FL_FREEZE) - - ret_label = asm.new_label('stack_ret') - - # String#+@ can only exist on T_STRING - stack_ret = ctx.stack_push(Type::TString) - - # If the string isn't frozen, we just return it. - asm.mov(stack_ret, :rax) # recv_opnd - asm.jz(ret_label) - - # Str is frozen - duplicate it - asm.mov(C_ARGS[0], :rax) # recv_opnd - asm.call(C.rb_str_dup) - asm.mov(stack_ret, C_RET) - - asm.write_label(ret_label) - - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_str_getbyte(jit, ctx, asm, argc, _known_recv_class) - return false if argc != 1 - asm.comment('rb_str_getbyte') - - index_opnd = ctx.stack_pop - str_opnd = ctx.stack_pop - asm.mov(C_ARGS[0], str_opnd) - asm.mov(C_ARGS[1], index_opnd) - asm.call(C.rb_str_getbyte) - - ret_opnd = ctx.stack_push(Type::Fixnum) - asm.mov(ret_opnd, C_RET) - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_ary_empty_p(jit, ctx, asm, argc, _known_recv_class) - array_reg = :rax - asm.mov(array_reg, ctx.stack_pop(1)) - jit_array_len(asm, array_reg, :rcx) - - asm.test(:rcx, :rcx) - asm.mov(:rax, Qfalse) - asm.mov(:rcx, Qtrue) - asm.cmovz(:rax, :rcx) - - out_opnd = ctx.stack_push(Type::UnknownImm) - asm.mov(out_opnd, :rax) - - return true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_ary_push(jit, ctx, asm, argc, _known_recv_class) - return false if argc != 1 - asm.comment('rb_ary_push') - - jit_prepare_routine_call(jit, ctx, asm) - - item_opnd = ctx.stack_pop - ary_opnd = ctx.stack_pop - asm.mov(C_ARGS[0], ary_opnd) - asm.mov(C_ARGS[1], item_opnd) - asm.call(C.rb_ary_push) - - ret_opnd = ctx.stack_push(Type::TArray) - asm.mov(ret_opnd, C_RET) - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_obj_respond_to(jit, ctx, asm, argc, known_recv_class) - # respond_to(:sym) or respond_to(:sym, true) - if argc != 1 && argc != 2 - return false - end - - if known_recv_class.nil? - return false - end - - recv_class = known_recv_class - - # Get the method_id from compile time. We will later add a guard against it. - mid_sym = jit.peek_at_stack(argc - 1) - unless static_symbol?(mid_sym) - return false - end - mid = C.rb_sym2id(mid_sym) - - # This represents the value of the "include_all" argument and whether it's known - allow_priv = if argc == 1 - # Default is false - false - else - # Get value from type information (may or may not be known) - ctx.get_opnd_type(StackOpnd[0]).known_truthy - end - - target_cme = C.rb_callable_method_entry_or_negative(recv_class, mid) - - # Should never be null, as in that case we will be returned a "negative CME" - assert_equal(false, target_cme.nil?) - - cme_def_type = C.UNDEFINED_METHOD_ENTRY_P(target_cme) ? C::VM_METHOD_TYPE_UNDEF : target_cme.def.type - - if cme_def_type == C::VM_METHOD_TYPE_REFINED - return false - end - - visibility = if cme_def_type == C::VM_METHOD_TYPE_UNDEF - C::METHOD_VISI_UNDEF - else - C.METHOD_ENTRY_VISI(target_cme) - end - - result = - case [visibility, allow_priv] - in C::METHOD_VISI_UNDEF, _ then Qfalse # No method => false - in C::METHOD_VISI_PUBLIC, _ then Qtrue # Public method => true regardless of include_all - in _, true then Qtrue # include_all => always true - else return false # not public and include_all not known, can't compile - end - - if result != Qtrue - # Only if respond_to_missing? hasn't been overridden - # In the future, we might want to jit the call to respond_to_missing? - unless Invariants.assume_method_basic_definition(jit, recv_class, C.idRespond_to_missing) - return false - end - end - - # Invalidate this block if method lookup changes for the method being queried. This works - # both for the case where a method does or does not exist, as for the latter we asked for a - # "negative CME" earlier. - Invariants.assume_method_lookup_stable(jit, target_cme) - - # Generate a side exit - side_exit = side_exit(jit, ctx) - - if argc == 2 - # pop include_all argument (we only use its type info) - ctx.stack_pop(1) - end - - sym_opnd = ctx.stack_pop(1) - _recv_opnd = ctx.stack_pop(1) - - # This is necessary because we have no guarantee that sym_opnd is a constant - asm.comment('guard known mid') - asm.mov(:rax, to_value(mid_sym)) - asm.cmp(sym_opnd, :rax) - asm.jne(side_exit) - - putobject(jit, ctx, asm, val: result) - - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_rb_f_block_given_p(jit, ctx, asm, argc, _known_recv_class) - asm.comment('block_given?') - - # Same as rb_vm_frame_block_handler - jit_get_lep(jit, asm, reg: :rax) - asm.mov(:rax, [:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler - - ctx.stack_pop(1) - out_opnd = ctx.stack_push(Type::UnknownImm) - - # Return `block_handler != VM_BLOCK_HANDLER_NONE` - asm.cmp(:rax, C::VM_BLOCK_HANDLER_NONE) - asm.mov(:rax, Qfalse) - asm.mov(:rcx, Qtrue) - asm.cmovne(:rax, :rcx) # block_given - asm.mov(out_opnd, :rax) - - true - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_thread_s_current(jit, ctx, asm, argc, _known_recv_class) - return false if argc != 0 - asm.comment('Thread.current') - ctx.stack_pop(1) - - # ec->thread_ptr - asm.mov(:rax, [EC, C.rb_execution_context_t.offsetof(:thread_ptr)]) - - # thread->self - asm.mov(:rax, [:rax, C.rb_thread_struct.offsetof(:self)]) - - stack_ret = ctx.stack_push(Type::UnknownHeap) - asm.mov(stack_ret, :rax) - true - end - - # - # Helpers - # - - def register_cfunc_codegen_funcs - # Specialization for C methods. See register_cfunc_method for details. - register_cfunc_method(BasicObject, :!, :jit_rb_obj_not) - - register_cfunc_method(NilClass, :nil?, :jit_rb_true) - register_cfunc_method(Kernel, :nil?, :jit_rb_false) - register_cfunc_method(Kernel, :is_a?, :jit_rb_kernel_is_a) - register_cfunc_method(Kernel, :kind_of?, :jit_rb_kernel_is_a) - register_cfunc_method(Kernel, :instance_of?, :jit_rb_kernel_instance_of) - - register_cfunc_method(BasicObject, :==, :jit_rb_obj_equal) - register_cfunc_method(BasicObject, :equal?, :jit_rb_obj_equal) - register_cfunc_method(BasicObject, :!=, :jit_rb_obj_not_equal) - register_cfunc_method(Kernel, :eql?, :jit_rb_obj_equal) - register_cfunc_method(Module, :==, :jit_rb_obj_equal) - register_cfunc_method(Module, :===, :jit_rb_mod_eqq) - register_cfunc_method(Symbol, :==, :jit_rb_obj_equal) - register_cfunc_method(Symbol, :===, :jit_rb_obj_equal) - register_cfunc_method(Integer, :==, :jit_rb_int_equal) - register_cfunc_method(Integer, :===, :jit_rb_int_equal) - - # rb_str_to_s() methods in string.c - register_cfunc_method(String, :empty?, :jit_rb_str_empty_p) - register_cfunc_method(String, :to_s, :jit_rb_str_to_s) - register_cfunc_method(String, :to_str, :jit_rb_str_to_s) - register_cfunc_method(String, :bytesize, :jit_rb_str_bytesize) - register_cfunc_method(String, :<<, :jit_rb_str_concat) - register_cfunc_method(String, :+@, :jit_rb_str_uplus) - - # rb_ary_empty_p() method in array.c - register_cfunc_method(Array, :empty?, :jit_rb_ary_empty_p) - - register_cfunc_method(Kernel, :respond_to?, :jit_obj_respond_to) - register_cfunc_method(Kernel, :block_given?, :jit_rb_f_block_given_p) - - # Thread.current - register_cfunc_method(C.rb_singleton_class(Thread), :current, :jit_thread_s_current) - - #--- - register_cfunc_method(Array, :<<, :jit_rb_ary_push) - register_cfunc_method(Integer, :*, :jit_rb_int_mul) - register_cfunc_method(Integer, :/, :jit_rb_int_div) - register_cfunc_method(Integer, :[], :jit_rb_int_aref) - register_cfunc_method(String, :getbyte, :jit_rb_str_getbyte) - end - - def register_cfunc_method(klass, mid_sym, func) - mid = C.rb_intern(mid_sym.to_s) - me = C.rb_method_entry_at(klass, mid) - - assert_equal(false, me.nil?) - - # Only cfuncs are supported - method_serial = me.def.method_serial - - @cfunc_codegen_table[method_serial] = method(func) - end - - def lookup_cfunc_codegen(cme_def) - @cfunc_codegen_table[cme_def.method_serial] - end - - def stack_swap(_jit, ctx, asm, offset0, offset1) - stack0_mem = ctx.stack_opnd(offset0) - stack1_mem = ctx.stack_opnd(offset1) - - mapping0 = ctx.get_opnd_mapping(StackOpnd[offset0]) - mapping1 = ctx.get_opnd_mapping(StackOpnd[offset1]) - - asm.mov(:rax, stack0_mem) - asm.mov(:rcx, stack1_mem) - asm.mov(stack0_mem, :rcx) - asm.mov(stack1_mem, :rax) - - ctx.set_opnd_mapping(StackOpnd[offset0], mapping1) - ctx.set_opnd_mapping(StackOpnd[offset1], mapping0) - end - - def jit_getlocal_generic(jit, ctx, asm, idx:, level:) - # Load environment pointer EP (level 0) from CFP - ep_reg = :rax - jit_get_ep(asm, level, reg: ep_reg) - - # Load the local from the block - # val = *(vm_get_ep(GET_EP(), level) - idx); - asm.mov(:rax, [ep_reg, -idx * C.VALUE.size]) - - # Write the local at SP - stack_top = if level == 0 - local_idx = ep_offset_to_local_idx(jit.iseq, idx) - ctx.stack_push_local(local_idx) - else - ctx.stack_push(Type::Unknown) - end - - asm.mov(stack_top, :rax) - KeepCompiling - end - - def jit_setlocal_generic(jit, ctx, asm, idx:, level:) - value_type = ctx.get_opnd_type(StackOpnd[0]) - - # Load environment pointer EP at level - ep_reg = :rax - jit_get_ep(asm, level, reg: ep_reg) - - # Write barriers may be required when VM_ENV_FLAG_WB_REQUIRED is set, however write barriers - # only affect heap objects being written. If we know an immediate value is being written we - # can skip this check. - unless value_type.imm? - # flags & VM_ENV_FLAG_WB_REQUIRED - flags_opnd = [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS] - asm.test(flags_opnd, C::VM_ENV_FLAG_WB_REQUIRED) - - # if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0 - asm.jnz(side_exit(jit, ctx)) - end - - if level == 0 - local_idx = ep_offset_to_local_idx(jit.iseq, idx) - ctx.set_local_type(local_idx, value_type) - end - - # Pop the value to write from the stack - stack_top = ctx.stack_pop(1) - - # Write the value at the environment pointer - asm.mov(:rcx, stack_top) - asm.mov([ep_reg, -(C.VALUE.size * idx)], :rcx) - - KeepCompiling - end - - # Compute the index of a local variable from its slot index - def ep_offset_to_local_idx(iseq, ep_offset) - # Layout illustration - # This is an array of VALUE - # | VM_ENV_DATA_SIZE | - # v v - # low addr <+-------+-------+-------+-------+------------------+ - # |local 0|local 1| ... |local n| .... | - # +-------+-------+-------+-------+------------------+ - # ^ ^ ^ ^ - # +-------+---local_table_size----+ cfp->ep--+ - # | | - # +------------------ep_offset---------------+ - # - # See usages of local_var_name() from iseq.c for similar calculation. - - # Equivalent of iseq->body->local_table_size - local_table_size = iseq.body.local_table_size - op = ep_offset - C::VM_ENV_DATA_SIZE - local_idx = local_table_size - op - 1 - assert_equal(true, local_idx >= 0 && local_idx < local_table_size) - local_idx - end - - # Compute the index of a local variable from its slot index - def slot_to_local_idx(iseq, slot_idx) - # Layout illustration - # This is an array of VALUE - # | VM_ENV_DATA_SIZE | - # v v - # low addr <+-------+-------+-------+-------+------------------+ - # |local 0|local 1| ... |local n| .... | - # +-------+-------+-------+-------+------------------+ - # ^ ^ ^ ^ - # +-------+---local_table_size----+ cfp->ep--+ - # | | - # +------------------slot_idx----------------+ - # - # See usages of local_var_name() from iseq.c for similar calculation. - - local_table_size = iseq.body.local_table_size - op = slot_idx - C::VM_ENV_DATA_SIZE - local_table_size - op - 1 - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def guard_object_is_heap(jit, ctx, asm, object, object_opnd, counter = nil) - object_type = ctx.get_opnd_type(object_opnd) - if object_type.heap? - return - end - - side_exit = side_exit(jit, ctx) - side_exit = counted_exit(side_exit, counter) if counter - - asm.comment('guard object is heap') - # Test that the object is not an immediate - asm.test(object, C::RUBY_IMMEDIATE_MASK) - asm.jnz(side_exit) - - # Test that the object is not false - asm.cmp(object, Qfalse) - asm.je(side_exit) - - if object_type.diff(Type::UnknownHeap) != TypeDiff::Incompatible - ctx.upgrade_opnd_type(object_opnd, Type::UnknownHeap) - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def guard_object_is_array(jit, ctx, asm, object_reg, flags_reg, object_opnd, counter = nil) - object_type = ctx.get_opnd_type(object_opnd) - if object_type.array? - return - end - - guard_object_is_heap(jit, ctx, asm, object_reg, object_opnd, counter) - - side_exit = side_exit(jit, ctx) - side_exit = counted_exit(side_exit, counter) if counter - - asm.comment('guard object is array') - # Pull out the type mask - asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)]) - asm.and(flags_reg, C::RUBY_T_MASK) - - # Compare the result with T_ARRAY - asm.cmp(flags_reg, C::RUBY_T_ARRAY) - asm.jne(side_exit) - - if object_type.diff(Type::TArray) != TypeDiff::Incompatible - ctx.upgrade_opnd_type(object_opnd, Type::TArray) - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def guard_object_is_string(jit, ctx, asm, object_reg, flags_reg, object_opnd, counter = nil) - object_type = ctx.get_opnd_type(object_opnd) - if object_type.string? - return - end - - guard_object_is_heap(jit, ctx, asm, object_reg, object_opnd, counter) - - side_exit = side_exit(jit, ctx) - side_exit = counted_exit(side_exit, counter) if counter - - asm.comment('guard object is string') - # Pull out the type mask - asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)]) - asm.and(flags_reg, C::RUBY_T_MASK) - - # Compare the result with T_STRING - asm.cmp(flags_reg, C::RUBY_T_STRING) - asm.jne(side_exit) - - if object_type.diff(Type::TString) != TypeDiff::Incompatible - ctx.upgrade_opnd_type(object_opnd, Type::TString) - end - end - - # clobbers object_reg - def guard_object_is_not_ruby2_keyword_hash(asm, object_reg, flags_reg, side_exit) - asm.comment('guard object is not ruby2 keyword hash') - - not_ruby2_keyword = asm.new_label('not_ruby2_keyword') - asm.test(object_reg, C::RUBY_IMMEDIATE_MASK) - asm.jnz(not_ruby2_keyword) - - asm.cmp(object_reg, Qfalse) - asm.je(not_ruby2_keyword) - - asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)]) - type_reg = object_reg - asm.mov(type_reg, flags_reg) - asm.and(type_reg, C::RUBY_T_MASK) - - asm.cmp(type_reg, C::RUBY_T_HASH) - asm.jne(not_ruby2_keyword) - - asm.test(flags_reg, C::RHASH_PASS_AS_KEYWORDS) - asm.jnz(side_exit) - - asm.write_label(not_ruby2_keyword) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_chain_guard(opcode, jit, ctx, asm, side_exit, limit: 20) - opcode => :je | :jne | :jnz | :jz - - if ctx.chain_depth < limit - deeper = ctx.dup - deeper.chain_depth += 1 - - branch_stub = BranchStub.new( - iseq: jit.iseq, - shape: Default, - target0: BranchTarget.new(ctx: deeper, pc: jit.pc), - ) - branch_stub.target0.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(deeper, ocb_asm, branch_stub, true) - @ocb.write(ocb_asm) - end - branch_stub.compile = compile_jit_chain_guard(branch_stub, opcode:) - branch_stub.compile.call(asm) - else - asm.public_send(opcode, side_exit) - end - end - - def compile_jit_chain_guard(branch_stub, opcode:) # Proc escapes arguments in memory - proc do |branch_asm| - # Not using `asm.comment` here since it's usually put before cmp/test before this. - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.public_send(opcode, branch_stub.target0.address) - end - end - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_guard_known_klass(jit, ctx, asm, known_klass, obj_opnd, insn_opnd, comptime_obj, side_exit, limit: 10) - # Only memory operand is supported for now - assert_equal(true, obj_opnd.is_a?(Array)) - - known_klass = C.to_value(known_klass) - val_type = ctx.get_opnd_type(insn_opnd) - if val_type.known_class == known_klass - # We already know from type information that this is a match - return - end - - # Touching this as Ruby could crash for FrozenCore - if known_klass == C.rb_cNilClass - assert(!val_type.heap?) - assert(val_type.unknown?) - - asm.comment('guard object is nil') - asm.cmp(obj_opnd, Qnil) - jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) - - ctx.upgrade_opnd_type(insn_opnd, Type::Nil) - elsif known_klass == C.rb_cTrueClass - assert(!val_type.heap?) - assert(val_type.unknown?) - - asm.comment('guard object is true') - asm.cmp(obj_opnd, Qtrue) - jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) - - ctx.upgrade_opnd_type(insn_opnd, Type::True) - elsif known_klass == C.rb_cFalseClass - assert(!val_type.heap?) - assert(val_type.unknown?) - - asm.comment('guard object is false') - asm.cmp(obj_opnd, Qfalse) - jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) - - ctx.upgrade_opnd_type(insn_opnd, Type::False) - elsif known_klass == C.rb_cInteger && fixnum?(comptime_obj) - # We will guard fixnum and bignum as though they were separate classes - # BIGNUM can be handled by the general else case below - assert(val_type.unknown?) - - asm.comment('guard object is fixnum') - asm.test(obj_opnd, C::RUBY_FIXNUM_FLAG) - jit_chain_guard(:jz, jit, ctx, asm, side_exit, limit:) - - ctx.upgrade_opnd_type(insn_opnd, Type::Fixnum) - elsif known_klass == C.rb_cSymbol && static_symbol?(comptime_obj) - assert(!val_type.heap?) - # We will guard STATIC vs DYNAMIC as though they were separate classes - # DYNAMIC symbols can be handled by the general else case below - if val_type != Type::ImmSymbol || !val_type.imm? - assert(val_type.unknown?) - - asm.comment('guard object is static symbol') - assert_equal(8, C::RUBY_SPECIAL_SHIFT) - asm.cmp(BytePtr[*obj_opnd], C::RUBY_SYMBOL_FLAG) - jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) - - ctx.upgrade_opnd_type(insn_opnd, Type::ImmSymbol) - end - elsif known_klass == C.rb_cFloat && flonum?(comptime_obj) - assert(!val_type.heap?) - if val_type != Type::Flonum || !val_type.imm? - assert(val_type.unknown?) - - # We will guard flonum vs heap float as though they were separate classes - asm.comment('guard object is flonum') - asm.mov(:rax, obj_opnd) - asm.and(:rax, C::RUBY_FLONUM_MASK) - asm.cmp(:rax, C::RUBY_FLONUM_FLAG) - jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) - - ctx.upgrade_opnd_type(insn_opnd, Type::Flonum) - end - elsif C.RCLASS_SINGLETON_P(known_klass) && comptime_obj == C.rb_class_attached_object(known_klass) - # Singleton classes are attached to one specific object, so we can - # avoid one memory access (and potentially the is_heap check) by - # looking for the expected object directly. - # Note that in case the sample instance has a singleton class that - # doesn't attach to the sample instance, it means the sample instance - # has an empty singleton class that hasn't been materialized yet. In - # this case, comparing against the sample instance doesn't guarantee - # that its singleton class is empty, so we can't avoid the memory - # access. As an example, `Object.new.singleton_class` is an object in - # this situation. - asm.comment('guard known object with singleton class') - asm.mov(:rax, to_value(comptime_obj)) - asm.cmp(obj_opnd, :rax) - jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) - elsif val_type == Type::CString && known_klass == C.rb_cString - # guard elided because the context says we've already checked - assert_equal(C.to_value(C.rb_class_of(comptime_obj)), C.rb_cString) - else - assert(!val_type.imm?) - - # Load memory to a register - asm.mov(:rax, obj_opnd) - obj_opnd = :rax - - # Check that the receiver is a heap object - # Note: if we get here, the class doesn't have immediate instances. - unless val_type.heap? - asm.comment('guard not immediate') - asm.test(obj_opnd, C::RUBY_IMMEDIATE_MASK) - jit_chain_guard(:jnz, jit, ctx, asm, side_exit, limit:) - asm.cmp(obj_opnd, Qfalse) - jit_chain_guard(:je, jit, ctx, asm, side_exit, limit:) - end - - # Bail if receiver class is different from known_klass - klass_opnd = [obj_opnd, C.RBasic.offsetof(:klass)] - asm.comment("guard known class #{known_klass}") - asm.mov(:rcx, known_klass) - asm.cmp(klass_opnd, :rcx) - jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) - - if known_klass == C.rb_cString - # Upgrading to Type::CString here is incorrect. - # The guard we put only checks RBASIC_CLASS(obj), - # which adding a singleton class can change. We - # additionally need to know the string is frozen - # to claim Type::CString. - ctx.upgrade_opnd_type(insn_opnd, Type::TString) - elsif known_klass == C.rb_cArray - ctx.upgrade_opnd_type(insn_opnd, Type::TArray) - end - end - end - - # @param jit [RubyVM::RJIT::JITState] - def two_fixnums_on_stack?(jit) - comptime_recv = jit.peek_at_stack(1) - comptime_arg = jit.peek_at_stack(0) - return fixnum?(comptime_recv) && fixnum?(comptime_arg) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def guard_two_fixnums(jit, ctx, asm) - # Get stack operands without popping them - arg1 = ctx.stack_opnd(0) - arg0 = ctx.stack_opnd(1) - - # Get the stack operand types - arg1_type = ctx.get_opnd_type(StackOpnd[0]) - arg0_type = ctx.get_opnd_type(StackOpnd[1]) - - if arg0_type.heap? || arg1_type.heap? - asm.comment('arg is heap object') - asm.jmp(side_exit(jit, ctx)) - return - end - - if arg0_type != Type::Fixnum && arg0_type.specific? - asm.comment('arg0 not fixnum') - asm.jmp(side_exit(jit, ctx)) - return - end - - if arg1_type != Type::Fixnum && arg1_type.specific? - asm.comment('arg1 not fixnum') - asm.jmp(side_exit(jit, ctx)) - return - end - - assert(!arg0_type.heap?) - assert(!arg1_type.heap?) - assert(arg0_type == Type::Fixnum || arg0_type.unknown?) - assert(arg1_type == Type::Fixnum || arg1_type.unknown?) - - # If not fixnums at run-time, fall back - if arg0_type != Type::Fixnum - asm.comment('guard arg0 fixnum') - asm.test(arg0, C::RUBY_FIXNUM_FLAG) - jit_chain_guard(:jz, jit, ctx, asm, side_exit(jit, ctx)) - end - if arg1_type != Type::Fixnum - asm.comment('guard arg1 fixnum') - asm.test(arg1, C::RUBY_FIXNUM_FLAG) - jit_chain_guard(:jz, jit, ctx, asm, side_exit(jit, ctx)) - end - - # Set stack types in context - ctx.upgrade_opnd_type(StackOpnd[0], Type::Fixnum) - ctx.upgrade_opnd_type(StackOpnd[1], Type::Fixnum) - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_fixnum_cmp(jit, ctx, asm, opcode:, bop:) - opcode => :cmovl | :cmovle | :cmovg | :cmovge - - unless jit.at_current_insn? - defer_compilation(jit, ctx, asm) - return EndBlock - end - - comptime_recv = jit.peek_at_stack(1) - comptime_obj = jit.peek_at_stack(0) - - if fixnum?(comptime_recv) && fixnum?(comptime_obj) - unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, bop) - return CantCompile - end - - # Check that both operands are fixnums - guard_two_fixnums(jit, ctx, asm) - - obj_opnd = ctx.stack_pop - recv_opnd = ctx.stack_pop - - asm.mov(:rax, obj_opnd) - asm.cmp(recv_opnd, :rax) - asm.mov(:rax, Qfalse) - asm.mov(:rcx, Qtrue) - asm.public_send(opcode, :rax, :rcx) - - dst_opnd = ctx.stack_push(Type::UnknownImm) - asm.mov(dst_opnd, :rax) - - KeepCompiling - else - opt_send_without_block(jit, ctx, asm) - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_equality_specialized(jit, ctx, asm, gen_eq) - # Create a side-exit to fall back to the interpreter - side_exit = side_exit(jit, ctx) - - a_opnd = ctx.stack_opnd(1) - b_opnd = ctx.stack_opnd(0) - - comptime_a = jit.peek_at_stack(1) - comptime_b = jit.peek_at_stack(0) - - if two_fixnums_on_stack?(jit) - unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_EQ) - return false - end - - guard_two_fixnums(jit, ctx, asm) - - asm.comment('check fixnum equality') - asm.mov(:rax, a_opnd) - asm.mov(:rcx, b_opnd) - asm.cmp(:rax, :rcx) - asm.mov(:rax, gen_eq ? Qfalse : Qtrue) - asm.mov(:rcx, gen_eq ? Qtrue : Qfalse) - asm.cmove(:rax, :rcx) - - # Push the output on the stack - ctx.stack_pop(2) - dst = ctx.stack_push(Type::UnknownImm) - asm.mov(dst, :rax) - - true - elsif C.rb_class_of(comptime_a) == String && C.rb_class_of(comptime_b) == String - unless Invariants.assume_bop_not_redefined(jit, C::STRING_REDEFINED_OP_FLAG, C::BOP_EQ) - # if overridden, emit the generic version - return false - end - - # Guard that a is a String - jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_a), a_opnd, StackOpnd[1], comptime_a, side_exit) - - equal_label = asm.new_label(:equal) - ret_label = asm.new_label(:ret) - - # If they are equal by identity, return true - asm.mov(:rax, a_opnd) - asm.mov(:rcx, b_opnd) - asm.cmp(:rax, :rcx) - asm.je(equal_label) - - # Otherwise guard that b is a T_STRING (from type info) or String (from runtime guard) - btype = ctx.get_opnd_type(StackOpnd[0]) - unless btype.string? - # Note: any T_STRING is valid here, but we check for a ::String for simplicity - # To pass a mutable static variable (rb_cString) requires an unsafe block - jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_b), b_opnd, StackOpnd[0], comptime_b, side_exit) - end - - asm.comment('call rb_str_eql_internal') - asm.mov(C_ARGS[0], a_opnd) - asm.mov(C_ARGS[1], b_opnd) - asm.call(gen_eq ? C.rb_str_eql_internal : C.rjit_str_neq_internal) - - # Push the output on the stack - ctx.stack_pop(2) - dst = ctx.stack_push(Type::UnknownImm) - asm.mov(dst, C_RET) - asm.jmp(ret_label) - - asm.write_label(equal_label) - asm.mov(dst, gen_eq ? Qtrue : Qfalse) - - asm.write_label(ret_label) - - true - else - false - end - end - - # NOTE: This clobbers :rax - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_prepare_routine_call(jit, ctx, asm) - jit.record_boundary_patch_point = true - jit_save_pc(jit, asm) - jit_save_sp(ctx, asm) - - # In case the routine calls Ruby methods, it can set local variables - # through Kernel#binding and other means. - ctx.clear_local_types - end - - # NOTE: This clobbers :rax - # @param jit [RubyVM::RJIT::JITState] - # @param asm [RubyVM::RJIT::Assembler] - def jit_save_pc(jit, asm, comment: 'save PC to CFP') - next_pc = jit.pc + jit.insn.len * C.VALUE.size # Use the next one for backtrace and side exits - asm.comment(comment) - asm.mov(:rax, next_pc) - asm.mov([CFP, C.rb_control_frame_t.offsetof(:pc)], :rax) - end - - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_save_sp(ctx, asm) - if ctx.sp_offset != 0 - asm.comment('save SP to CFP') - asm.lea(SP, ctx.sp_opnd) - asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], SP) - ctx.sp_offset = 0 - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jump_to_next_insn(jit, ctx, asm) - reset_depth = ctx.dup - reset_depth.chain_depth = 0 - - next_pc = jit.pc + jit.insn.len * C.VALUE.size - - # We are at the end of the current instruction. Record the boundary. - if jit.record_boundary_patch_point - exit_pos = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_side_exit(next_pc, ctx, ocb_asm) - @ocb.write(ocb_asm) - end - Invariants.record_global_inval_patch(asm, exit_pos) - jit.record_boundary_patch_point = false - end - - jit_direct_jump(jit.iseq, next_pc, reset_depth, asm, comment: 'jump_to_next_insn') - end - - # rb_vm_check_ints - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_check_ints(jit, ctx, asm) - asm.comment('RUBY_VM_CHECK_INTS(ec)') - asm.mov(:eax, DwordPtr[EC, C.rb_execution_context_t.offsetof(:interrupt_flag)]) - asm.test(:eax, :eax) - asm.jnz(side_exit(jit, ctx)) - end - - # See get_lvar_level in compile.c - def get_lvar_level(iseq) - level = 0 - while iseq.to_i != iseq.body.local_iseq.to_i - level += 1 - iseq = iseq.body.parent_iseq - end - return level - end - - # GET_LEP - # @param jit [RubyVM::RJIT::JITState] - # @param asm [RubyVM::RJIT::Assembler] - def jit_get_lep(jit, asm, reg:) - level = get_lvar_level(jit.iseq) - jit_get_ep(asm, level, reg:) - end - - # vm_get_ep - # @param asm [RubyVM::RJIT::Assembler] - def jit_get_ep(asm, level, reg:) - asm.mov(reg, [CFP, C.rb_control_frame_t.offsetof(:ep)]) - level.times do - # GET_PREV_EP: ep[VM_ENV_DATA_INDEX_SPECVAL] & ~0x03 - asm.mov(reg, [reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) - asm.and(reg, ~0x03) - end - end - - # vm_getivar - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_getivar(jit, ctx, asm, comptime_obj, ivar_id, obj_opnd, obj_yarv_opnd) - side_exit = side_exit(jit, ctx) - starting_ctx = ctx.dup # copy for jit_chain_guard - - # Guard not special const - if C::SPECIAL_CONST_P(comptime_obj) - asm.incr_counter(:getivar_special_const) - return CantCompile - end - - case C::BUILTIN_TYPE(comptime_obj) - when C::T_OBJECT - # This is the only supported case for now (ROBJECT_IVPTR) - else - # General case. Call rb_ivar_get(). - # VALUE rb_ivar_get(VALUE obj, ID id) - asm.comment('call rb_ivar_get()') - asm.mov(C_ARGS[0], obj_opnd ? obj_opnd : [CFP, C.rb_control_frame_t.offsetof(:self)]) - asm.mov(C_ARGS[1], ivar_id) - - # The function could raise exceptions. - jit_prepare_routine_call(jit, ctx, asm) # clobbers obj_opnd and :rax - - asm.call(C.rb_ivar_get) - - if obj_opnd # attr_reader - ctx.stack_pop - end - - # Push the ivar on the stack - out_opnd = ctx.stack_push(Type::Unknown) - asm.mov(out_opnd, C_RET) - - # Jump to next instruction. This allows guard chains to share the same successor. - jump_to_next_insn(jit, ctx, asm) - return EndBlock - end - - asm.mov(:rax, obj_opnd ? obj_opnd : [CFP, C.rb_control_frame_t.offsetof(:self)]) - guard_object_is_heap(jit, ctx, asm, :rax, obj_yarv_opnd, :getivar_not_heap) - - shape_id = C.rb_shape_get_shape_id(comptime_obj) - if shape_id == C::OBJ_TOO_COMPLEX_SHAPE_ID - asm.incr_counter(:getivar_too_complex) - return CantCompile - end - - asm.comment('guard shape') - asm.cmp(DwordPtr[:rax, C.rb_shape_id_offset], shape_id) - jit_chain_guard(:jne, jit, starting_ctx, asm, counted_exit(side_exit, :getivar_megamorphic)) - - if obj_opnd - ctx.stack_pop # pop receiver for attr_reader - end - - index = C.rb_shape_get_iv_index(shape_id, ivar_id) - # If there is no IVAR index, then the ivar was undefined - # when we entered the compiler. That means we can just return - # nil for this shape + iv name - if index.nil? - stack_opnd = ctx.stack_push(Type::Nil) - val_opnd = Qnil - else - asm.comment('ROBJECT_IVPTR') - if C::FL_TEST_RAW(comptime_obj, C::ROBJECT_EMBED) - # Access embedded array - asm.mov(:rax, [:rax, C.RObject.offsetof(:as, :ary) + (index * C.VALUE.size)]) - else - # Pull out an ivar table on heap - asm.mov(:rax, [:rax, C.RObject.offsetof(:as, :heap, :ivptr)]) - # Read the table - asm.mov(:rax, [:rax, index * C.VALUE.size]) - end - stack_opnd = ctx.stack_push(Type::Unknown) - val_opnd = :rax - end - asm.mov(stack_opnd, val_opnd) - - # Let guard chains share the same successor - jump_to_next_insn(jit, ctx, asm) - EndBlock - end - - def jit_write_iv(asm, comptime_receiver, recv_reg, temp_reg, ivar_index, set_value, needs_extension) - # Compile time self is embedded and the ivar index lands within the object - embed_test_result = C::FL_TEST_RAW(comptime_receiver, C::ROBJECT_EMBED) && !needs_extension - - if embed_test_result - # Find the IV offset - offs = C.RObject.offsetof(:as, :ary) + ivar_index * C.VALUE.size - - # Write the IV - asm.comment('write IV') - asm.mov(temp_reg, set_value) - asm.mov([recv_reg, offs], temp_reg) - else - # Compile time value is *not* embedded. - - # Get a pointer to the extended table - asm.mov(recv_reg, [recv_reg, C.RObject.offsetof(:as, :heap, :ivptr)]) - - # Write the ivar in to the extended table - asm.comment("write IV"); - asm.mov(temp_reg, set_value) - asm.mov([recv_reg, C.VALUE.size * ivar_index], temp_reg) - end - end - - # vm_caller_setup_arg_block: Handle VM_CALL_ARGS_BLOCKARG cases. - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def guard_block_arg(jit, ctx, asm, calling) - if calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 - block_arg_type = ctx.get_opnd_type(StackOpnd[0]) - case block_arg_type - in Type::Nil - calling.block_handler = C::VM_BLOCK_HANDLER_NONE - in Type::BlockParamProxy - calling.block_handler = C.rb_block_param_proxy - else - asm.incr_counter(:send_block_arg) - return CantCompile - end - end - end - - # vm_search_method - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_search_method(jit, ctx, asm, mid, calling) - assert_equal(true, jit.at_current_insn?) - - # Generate a side exit - side_exit = side_exit(jit, ctx) - - # kw_splat is not supported yet - if calling.flags & C::VM_CALL_KW_SPLAT != 0 - asm.incr_counter(:send_kw_splat) - return CantCompile - end - - # Get a compile-time receiver and its class - recv_idx = calling.argc + (calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 ? 1 : 0) # blockarg is not popped yet - recv_idx += calling.send_shift - comptime_recv = jit.peek_at_stack(recv_idx) - comptime_recv_klass = C.rb_class_of(comptime_recv) - - # Guard the receiver class (part of vm_search_method_fastpath) - recv_opnd = ctx.stack_opnd(recv_idx) - megamorphic_exit = counted_exit(side_exit, :send_klass_megamorphic) - jit_guard_known_klass(jit, ctx, asm, comptime_recv_klass, recv_opnd, StackOpnd[recv_idx], comptime_recv, megamorphic_exit) - - # Do method lookup (vm_cc_cme(cc) != NULL) - cme = C.rb_callable_method_entry(comptime_recv_klass, mid) - if cme.nil? - asm.incr_counter(:send_missing_cme) - return CantCompile # We don't support vm_call_method_name - end - - # Invalidate on redefinition (part of vm_search_method_fastpath) - Invariants.assume_method_lookup_stable(jit, cme) - - return cme, comptime_recv_klass - end - - # vm_call_general - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_call_general(jit, ctx, asm, mid, calling, cme, known_recv_class) - jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class) - end - - # vm_call_method - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - # @param send_shift [Integer] The number of shifts needed for VM_CALL_OPT_SEND - def jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class) - # The main check of vm_call_method before vm_call_method_each_type - case C::METHOD_ENTRY_VISI(cme) - in C::METHOD_VISI_PUBLIC - # You can always call public methods - in C::METHOD_VISI_PRIVATE - # Allow only callsites without a receiver - if calling.flags & C::VM_CALL_FCALL == 0 - asm.incr_counter(:send_private) - return CantCompile - end - in C::METHOD_VISI_PROTECTED - # If the method call is an FCALL, it is always valid - if calling.flags & C::VM_CALL_FCALL == 0 - # otherwise we need an ancestry check to ensure the receiver is valid to be called as protected - jit_protected_callee_ancestry_guard(asm, cme, side_exit(jit, ctx)) - end - end - - # Get a compile-time receiver - recv_idx = calling.argc + (calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 ? 1 : 0) # blockarg is not popped yet - recv_idx += calling.send_shift - comptime_recv = jit.peek_at_stack(recv_idx) - recv_opnd = ctx.stack_opnd(recv_idx) - - jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) - end - - # Generate ancestry guard for protected callee. - # Calls to protected callees only go through when self.is_a?(klass_that_defines_the_callee). - def jit_protected_callee_ancestry_guard(asm, cme, side_exit) - # See vm_call_method(). - def_class = cme.defined_class - # Note: PC isn't written to current control frame as rb_is_kind_of() shouldn't raise. - # VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass); - - asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:self)]) - asm.mov(C_ARGS[1], to_value(def_class)) - asm.call(C.rb_obj_is_kind_of) - asm.test(C_RET, C_RET) - asm.jz(counted_exit(side_exit, :send_protected_check_failed)) - end - - # vm_call_method_each_type - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) - case cme.def.type - in C::VM_METHOD_TYPE_ISEQ - iseq = def_iseq_ptr(cme.def) - jit_call_iseq(jit, ctx, asm, cme, calling, iseq) - in C::VM_METHOD_TYPE_NOTIMPLEMENTED - asm.incr_counter(:send_notimplemented) - return CantCompile - in C::VM_METHOD_TYPE_CFUNC - jit_call_cfunc(jit, ctx, asm, cme, calling, known_recv_class:) - in C::VM_METHOD_TYPE_ATTRSET - jit_call_attrset(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd) - in C::VM_METHOD_TYPE_IVAR - jit_call_ivar(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd) - in C::VM_METHOD_TYPE_MISSING - asm.incr_counter(:send_missing) - return CantCompile - in C::VM_METHOD_TYPE_BMETHOD - jit_call_bmethod(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) - in C::VM_METHOD_TYPE_ALIAS - jit_call_alias(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) - in C::VM_METHOD_TYPE_OPTIMIZED - jit_call_optimized(jit, ctx, asm, cme, calling, known_recv_class) - in C::VM_METHOD_TYPE_UNDEF - asm.incr_counter(:send_undef) - return CantCompile - in C::VM_METHOD_TYPE_ZSUPER - asm.incr_counter(:send_zsuper) - return CantCompile - in C::VM_METHOD_TYPE_REFINED - asm.incr_counter(:send_refined) - return CantCompile - end - end - - # vm_call_iseq_setup - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_call_iseq(jit, ctx, asm, cme, calling, iseq, frame_type: nil, prev_ep: nil) - argc = calling.argc - flags = calling.flags - send_shift = calling.send_shift - - # When you have keyword arguments, there is an extra object that gets - # placed on the stack the represents a bitmap of the keywords that were not - # specified at the call site. We need to keep track of the fact that this - # value is present on the stack in order to properly set up the callee's - # stack pointer. - doing_kw_call = iseq.body.param.flags.has_kw - supplying_kws = flags & C::VM_CALL_KWARG != 0 - - if flags & C::VM_CALL_TAILCALL != 0 - # We can't handle tailcalls - asm.incr_counter(:send_tailcall) - return CantCompile - end - - # No support for callees with these parameters yet as they require allocation - # or complex handling. - if iseq.body.param.flags.has_post - asm.incr_counter(:send_iseq_has_opt) - return CantCompile - end - if iseq.body.param.flags.has_kwrest - asm.incr_counter(:send_iseq_has_kwrest) - return CantCompile - end - - # In order to handle backwards compatibility between ruby 3 and 2 - # ruby2_keywords was introduced. It is called only on methods - # with splat and changes they way they handle them. - # We are just going to not compile these. - # https://www.rubydoc.info/stdlib/core/Proc:ruby2_keywords - if iseq.body.param.flags.ruby2_keywords && flags & C::VM_CALL_ARGS_SPLAT != 0 - asm.incr_counter(:send_iseq_ruby2_keywords) - return CantCompile - end - - iseq_has_rest = iseq.body.param.flags.has_rest - if iseq_has_rest && calling.block_handler == :captured - asm.incr_counter(:send_iseq_has_rest_and_captured) - return CantCompile - end - - if iseq_has_rest && iseq.body.param.flags.has_kw && supplying_kws - asm.incr_counter(:send_iseq_has_rest_and_kw_supplied) - return CantCompile - end - - # If we have keyword arguments being passed to a callee that only takes - # positionals, then we need to allocate a hash. For now we're going to - # call that too complex and bail. - if supplying_kws && !iseq.body.param.flags.has_kw - asm.incr_counter(:send_iseq_has_no_kw) - return CantCompile - end - - # If we have a method accepting no kwargs (**nil), exit if we have passed - # it any kwargs. - if supplying_kws && iseq.body.param.flags.accepts_no_kwarg - asm.incr_counter(:send_iseq_accepts_no_kwarg) - return CantCompile - end - - # For computing number of locals to set up for the callee - num_params = iseq.body.param.size - - # Block parameter handling. This mirrors setup_parameters_complex(). - if iseq.body.param.flags.has_block - if iseq.body.local_iseq.to_i == iseq.to_i - num_params -= 1 - else - # In this case (param.flags.has_block && local_iseq != iseq), - # the block argument is setup as a local variable and requires - # materialization (allocation). Bail. - asm.incr_counter(:send_iseq_materialized_block) - return CantCompile - end - end - - if flags & C::VM_CALL_ARGS_SPLAT != 0 && flags & C::VM_CALL_ZSUPER != 0 - # zsuper methods are super calls without any arguments. - # They are also marked as splat, but don't actually have an array - # they pull arguments from, instead we need to change to call - # a different method with the current stack. - asm.incr_counter(:send_iseq_zsuper) - return CantCompile - end - - start_pc_offset = 0 - required_num = iseq.body.param.lead_num - - # This struct represents the metadata about the caller-specified - # keyword arguments. - kw_arg = calling.kwarg - kw_arg_num = if kw_arg.nil? - 0 - else - kw_arg.keyword_len - end - - # Arity handling and optional parameter setup - opts_filled = argc - required_num - kw_arg_num - opt_num = iseq.body.param.opt_num - opts_missing = opt_num - opts_filled - - if doing_kw_call && flags & C::VM_CALL_ARGS_SPLAT != 0 - asm.incr_counter(:send_iseq_splat_with_kw) - return CantCompile - end - - if flags & C::VM_CALL_KW_SPLAT != 0 - asm.incr_counter(:send_iseq_kw_splat) - return CantCompile - end - - if iseq_has_rest && opt_num != 0 - asm.incr_counter(:send_iseq_has_rest_and_optional) - return CantCompile - end - - if opts_filled < 0 && flags & C::VM_CALL_ARGS_SPLAT == 0 - # Too few arguments and no splat to make up for it - asm.incr_counter(:send_iseq_arity_error) - return CantCompile - end - - if opts_filled > opt_num && !iseq_has_rest - # Too many arguments and no place to put them (i.e. rest arg) - asm.incr_counter(:send_iseq_arity_error) - return CantCompile - end - - block_arg = flags & C::VM_CALL_ARGS_BLOCKARG != 0 - - # Guard block_arg_type - if guard_block_arg(jit, ctx, asm, calling) == CantCompile - return CantCompile - end - - # If we have unfilled optional arguments and keyword arguments then we - # would need to adjust the arguments location to account for that. - # For now we aren't handling this case. - if doing_kw_call && opts_missing > 0 - asm.incr_counter(:send_iseq_missing_optional_kw) - return CantCompile - end - - # We will handle splat case later - if opt_num > 0 && flags & C::VM_CALL_ARGS_SPLAT == 0 - num_params -= opts_missing - start_pc_offset = iseq.body.param.opt_table[opts_filled] - end - - if doing_kw_call - # Here we're calling a method with keyword arguments and specifying - # keyword arguments at this call site. - - # This struct represents the metadata about the callee-specified - # keyword parameters. - keyword = iseq.body.param.keyword - keyword_num = keyword.num - keyword_required_num = keyword.required_num - - required_kwargs_filled = 0 - - if keyword_num > 30 - # We have so many keywords that (1 << num) encoded as a FIXNUM - # (which shifts it left one more) no longer fits inside a 32-bit - # immediate. - asm.incr_counter(:send_iseq_too_many_kwargs) - return CantCompile - end - - # Check that the kwargs being passed are valid - if supplying_kws - # This is the list of keyword arguments that the callee specified - # in its initial declaration. - # SAFETY: see compile.c for sizing of this slice. - callee_kwargs = keyword_num.times.map { |i| keyword.table[i] } - - # Here we're going to build up a list of the IDs that correspond to - # the caller-specified keyword arguments. If they're not in the - # same order as the order specified in the callee declaration, then - # we're going to need to generate some code to swap values around - # on the stack. - caller_kwargs = [] - kw_arg.keyword_len.times do |kwarg_idx| - sym = C.to_ruby(kw_arg[:keywords][kwarg_idx]) - caller_kwargs << C.rb_sym2id(sym) - end - - # First, we're going to be sure that the names of every - # caller-specified keyword argument correspond to a name in the - # list of callee-specified keyword parameters. - caller_kwargs.each do |caller_kwarg| - search_result = callee_kwargs.map.with_index.find { |kwarg, _| kwarg == caller_kwarg } - - case search_result - in nil - # If the keyword was never found, then we know we have a - # mismatch in the names of the keyword arguments, so we need to - # bail. - asm.incr_counter(:send_iseq_kwargs_mismatch) - return CantCompile - in _, callee_idx if callee_idx < keyword_required_num - # Keep a count to ensure all required kwargs are specified - required_kwargs_filled += 1 - else - end - end - end - assert_equal(true, required_kwargs_filled <= keyword_required_num) - if required_kwargs_filled != keyword_required_num - asm.incr_counter(:send_iseq_kwargs_mismatch) - return CantCompile - end - end - - # Check if we need the arg0 splat handling of vm_callee_setup_block_arg - arg_setup_block = (calling.block_handler == :captured) # arg_setup_type: arg_setup_block (invokeblock) - block_arg0_splat = arg_setup_block && argc == 1 && - (iseq.body.param.flags.has_lead || opt_num > 1) && - !iseq.body.param.flags.ambiguous_param0 - if block_arg0_splat - # If block_arg0_splat, we still need side exits after splat, but - # doing push_splat_args here disallows it. So bail out. - if flags & C::VM_CALL_ARGS_SPLAT != 0 && !iseq_has_rest - asm.incr_counter(:invokeblock_iseq_arg0_args_splat) - return CantCompile - end - # The block_arg0_splat implementation is for the rb_simple_iseq_p case, - # but doing_kw_call means it's not a simple ISEQ. - if doing_kw_call - asm.incr_counter(:invokeblock_iseq_arg0_has_kw) - return CantCompile - end - # The block_arg0_splat implementation cannot deal with optional parameters. - # This is a setup_parameters_complex() situation and interacts with the - # starting position of the callee. - if opt_num > 1 - asm.incr_counter(:invokeblock_iseq_arg0_optional) - return CantCompile - end - end - if flags & C::VM_CALL_ARGS_SPLAT != 0 && !iseq_has_rest - array = jit.peek_at_stack(block_arg ? 1 : 0) - splat_array_length = if array.nil? - 0 - else - array.length - end - - if opt_num == 0 && required_num != splat_array_length + argc - 1 - asm.incr_counter(:send_iseq_splat_arity_error) - return CantCompile - end - end - - # Don't compile forwardable iseqs - if iseq.body.param.flags.forwardable - return CantCompile - end - - # We will not have CantCompile from here. - - if block_arg - ctx.stack_pop(1) - end - - if calling.block_handler == C::VM_BLOCK_HANDLER_NONE && iseq.body.builtin_attrs & C::BUILTIN_ATTR_LEAF != 0 - if jit_leaf_builtin_func(jit, ctx, asm, flags, iseq) - return KeepCompiling - end - end - - # Number of locals that are not parameters - num_locals = iseq.body.local_table_size - num_params - - # Stack overflow check - # Note that vm_push_frame checks it against a decremented cfp, hence the multiply by 2. - # #define CHECK_VM_STACK_OVERFLOW0(cfp, sp, margin) - asm.comment('stack overflow check') - locals_offs = C.VALUE.size * (num_locals + iseq.body.stack_max) + 2 * C.rb_control_frame_t.size - asm.lea(:rax, ctx.sp_opnd(locals_offs)) - asm.cmp(CFP, :rax) - asm.jbe(counted_exit(side_exit(jit, ctx), :send_stackoverflow)) - - # push_splat_args does stack manipulation so we can no longer side exit - if splat_array_length - remaining_opt = (opt_num + required_num) - (splat_array_length + (argc - 1)) - - if opt_num > 0 - # We are going to jump to the correct offset based on how many optional - # params are remaining. - offset = opt_num - remaining_opt - start_pc_offset = iseq.body.param.opt_table[offset] - end - # We are going to assume that the splat fills - # all the remaining arguments. In the generated code - # we test if this is true and if not side exit. - argc = argc - 1 + splat_array_length + remaining_opt - push_splat_args(splat_array_length, jit, ctx, asm) - - remaining_opt.times do - # We need to push nil for the optional arguments - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, Qnil) - end - end - - # This is a .send call and we need to adjust the stack - if flags & C::VM_CALL_OPT_SEND != 0 - handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) - end - - if iseq_has_rest - # We are going to allocate so setting pc and sp. - jit_save_pc(jit, asm) # clobbers rax - jit_save_sp(ctx, asm) - - if flags & C::VM_CALL_ARGS_SPLAT != 0 - non_rest_arg_count = argc - 1 - # We start by dupping the array because someone else might have - # a reference to it. - array = ctx.stack_pop(1) - asm.mov(C_ARGS[0], array) - asm.call(C.rb_ary_dup) - array = C_RET - if non_rest_arg_count > required_num - # If we have more arguments than required, we need to prepend - # the items from the stack onto the array. - diff = (non_rest_arg_count - required_num) - - # diff is >0 so no need to worry about null pointer - asm.comment('load pointer to array elements') - offset_magnitude = C.VALUE.size * diff - values_opnd = ctx.sp_opnd(-offset_magnitude) - values_ptr = :rcx - asm.lea(values_ptr, values_opnd) - - asm.comment('prepend stack values to rest array') - asm.mov(C_ARGS[0], diff) - asm.mov(C_ARGS[1], values_ptr) - asm.mov(C_ARGS[2], array) - asm.call(C.rb_ary_unshift_m) - ctx.stack_pop(diff) - - stack_ret = ctx.stack_push(Type::TArray) - asm.mov(stack_ret, C_RET) - # We now should have the required arguments - # and an array of all the rest arguments - argc = required_num + 1 - elsif non_rest_arg_count < required_num - # If we have fewer arguments than required, we need to take some - # from the array and move them to the stack. - diff = (required_num - non_rest_arg_count) - # This moves the arguments onto the stack. But it doesn't modify the array. - move_rest_args_to_stack(array, diff, jit, ctx, asm) - - # We will now slice the array to give us a new array of the correct size - asm.mov(C_ARGS[0], array) - asm.mov(C_ARGS[1], diff) - asm.call(C.rjit_rb_ary_subseq_length) - stack_ret = ctx.stack_push(Type::TArray) - asm.mov(stack_ret, C_RET) - - # We now should have the required arguments - # and an array of all the rest arguments - argc = required_num + 1 - else - # The arguments are equal so we can just push to the stack - assert_equal(non_rest_arg_count, required_num) - stack_ret = ctx.stack_push(Type::TArray) - asm.mov(stack_ret, array) - end - else - assert_equal(true, argc >= required_num) - n = (argc - required_num) - argc = required_num + 1 - # If n is 0, then elts is never going to be read, so we can just pass null - if n == 0 - values_ptr = 0 - else - asm.comment('load pointer to array elements') - offset_magnitude = C.VALUE.size * n - values_opnd = ctx.sp_opnd(-offset_magnitude) - values_ptr = :rcx - asm.lea(values_ptr, values_opnd) - end - - asm.mov(C_ARGS[0], EC) - asm.mov(C_ARGS[1], n) - asm.mov(C_ARGS[2], values_ptr) - asm.call(C.rb_ec_ary_new_from_values) - - ctx.stack_pop(n) - stack_ret = ctx.stack_push(Type::TArray) - asm.mov(stack_ret, C_RET) - end - end - - if doing_kw_call - # Here we're calling a method with keyword arguments and specifying - # keyword arguments at this call site. - - # Number of positional arguments the callee expects before the first - # keyword argument - args_before_kw = required_num + opt_num - - # This struct represents the metadata about the caller-specified - # keyword arguments. - ci_kwarg = calling.kwarg - caller_keyword_len = if ci_kwarg.nil? - 0 - else - ci_kwarg.keyword_len - end - - # This struct represents the metadata about the callee-specified - # keyword parameters. - keyword = iseq.body.param.keyword - - asm.comment('keyword args') - - # This is the list of keyword arguments that the callee specified - # in its initial declaration. - callee_kwargs = keyword.table - total_kwargs = keyword.num - - # Here we're going to build up a list of the IDs that correspond to - # the caller-specified keyword arguments. If they're not in the - # same order as the order specified in the callee declaration, then - # we're going to need to generate some code to swap values around - # on the stack. - caller_kwargs = [] - - caller_keyword_len.times do |kwarg_idx| - sym = C.to_ruby(ci_kwarg[:keywords][kwarg_idx]) - caller_kwargs << C.rb_sym2id(sym) - end - kwarg_idx = caller_keyword_len - - unspecified_bits = 0 - - keyword_required_num = keyword.required_num - (keyword_required_num...total_kwargs).each do |callee_idx| - already_passed = false - callee_kwarg = callee_kwargs[callee_idx] - - caller_keyword_len.times do |caller_idx| - if caller_kwargs[caller_idx] == callee_kwarg - already_passed = true - break - end - end - - unless already_passed - # Reserve space on the stack for each default value we'll be - # filling in (which is done in the next loop). Also increments - # argc so that the callee's SP is recorded correctly. - argc += 1 - default_arg = ctx.stack_push(Type::Unknown) - - # callee_idx - keyword->required_num is used in a couple of places below. - req_num = keyword.required_num - extra_args = callee_idx - req_num - - # VALUE default_value = keyword->default_values[callee_idx - keyword->required_num]; - default_value = keyword.default_values[extra_args] - - if default_value == Qundef - # Qundef means that this value is not constant and must be - # recalculated at runtime, so we record it in unspecified_bits - # (Qnil is then used as a placeholder instead of Qundef). - unspecified_bits |= 0x01 << extra_args - default_value = Qnil - end - - asm.mov(:rax, default_value) - asm.mov(default_arg, :rax) - - caller_kwargs[kwarg_idx] = callee_kwarg - kwarg_idx += 1 - end - end - - assert_equal(kwarg_idx, total_kwargs) - - # Next, we're going to loop through every keyword that was - # specified by the caller and make sure that it's in the correct - # place. If it's not we're going to swap it around with another one. - total_kwargs.times do |kwarg_idx| - callee_kwarg = callee_kwargs[kwarg_idx] - - # If the argument is already in the right order, then we don't - # need to generate any code since the expected value is already - # in the right place on the stack. - if callee_kwarg == caller_kwargs[kwarg_idx] - next - end - - # In this case the argument is not in the right place, so we - # need to find its position where it _should_ be and swap with - # that location. - ((kwarg_idx + 1)...total_kwargs).each do |swap_idx| - if callee_kwarg == caller_kwargs[swap_idx] - # First we're going to generate the code that is going - # to perform the actual swapping at runtime. - offset0 = argc - 1 - swap_idx - args_before_kw - offset1 = argc - 1 - kwarg_idx - args_before_kw - stack_swap(jit, ctx, asm, offset0, offset1) - - # Next we're going to do some bookkeeping on our end so - # that we know the order that the arguments are - # actually in now. - caller_kwargs[kwarg_idx], caller_kwargs[swap_idx] = - caller_kwargs[swap_idx], caller_kwargs[kwarg_idx] - - break - end - end - end - - # Keyword arguments cause a special extra local variable to be - # pushed onto the stack that represents the parameters that weren't - # explicitly given a value and have a non-constant default. - asm.mov(ctx.stack_opnd(-1), C.to_value(unspecified_bits)) - end - - # Same as vm_callee_setup_block_arg_arg0_check and vm_callee_setup_block_arg_arg0_splat - # on vm_callee_setup_block_arg for arg_setup_block. This is done after CALLER_SETUP_ARG - # and CALLER_REMOVE_EMPTY_KW_SPLAT, so this implementation is put here. This may need - # side exits, so you still need to allow side exits here if block_arg0_splat is true. - # Note that you can't have side exits after this arg0 splat. - if block_arg0_splat - asm.incr_counter(:send_iseq_block_arg0_splat) - return CantCompile - end - - # Create a context for the callee - callee_ctx = Context.new - - # Set the argument types in the callee's context - argc.times do |arg_idx| - stack_offs = argc - arg_idx - 1 - arg_type = ctx.get_opnd_type(StackOpnd[stack_offs]) - callee_ctx.set_local_type(arg_idx, arg_type) - end - - recv_type = if calling.block_handler == :captured - Type::Unknown # we don't track the type information of captured->self for now - else - ctx.get_opnd_type(StackOpnd[argc]) - end - callee_ctx.upgrade_opnd_type(SelfOpnd, recv_type) - - # Setup the new frame - frame_type ||= C::VM_FRAME_MAGIC_METHOD | C::VM_ENV_FLAG_LOCAL - jit_push_frame( - jit, ctx, asm, cme, flags, argc, frame_type, calling.block_handler, - iseq: iseq, - local_size: num_locals, - stack_max: iseq.body.stack_max, - prev_ep:, - doing_kw_call:, - ) - - # Directly jump to the entry point of the callee - pc = (iseq.body.iseq_encoded + start_pc_offset).to_i - jit_direct_jump(iseq, pc, callee_ctx, asm) - - EndBlock - end - - def jit_leaf_builtin_func(jit, ctx, asm, flags, iseq) - builtin_func = builtin_function(iseq) - if builtin_func.nil? - return false - end - - # this is a .send call not currently supported for builtins - if flags & C::VM_CALL_OPT_SEND != 0 - return false - end - - builtin_argc = builtin_func.argc - if builtin_argc + 1 >= C_ARGS.size - return false - end - - asm.comment('inlined leaf builtin') - - # The callee may allocate, e.g. Integer#abs on a Bignum. - # Save SP for GC, save PC for allocation tracing, and prepare - # for global invalidation after GC's VM lock contention. - jit_prepare_routine_call(jit, ctx, asm) - - # Call the builtin func (ec, recv, arg1, arg2, ...) - asm.mov(C_ARGS[0], EC) - - # Copy self and arguments - (0..builtin_argc).each do |i| - stack_opnd = ctx.stack_opnd(builtin_argc - i) - asm.mov(C_ARGS[i + 1], stack_opnd) - end - ctx.stack_pop(builtin_argc + 1) - asm.call(builtin_func.func_ptr) - - # Push the return value - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - return true - end - - # vm_call_cfunc - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_call_cfunc(jit, ctx, asm, cme, calling, known_recv_class: nil) - argc = calling.argc - flags = calling.flags - - cfunc = cme.def.body.cfunc - cfunc_argc = cfunc.argc - - # If the function expects a Ruby array of arguments - if cfunc_argc < 0 && cfunc_argc != -1 - asm.incr_counter(:send_cfunc_ruby_array_varg) - return CantCompile - end - - # We aren't handling a vararg cfuncs with splat currently. - if flags & C::VM_CALL_ARGS_SPLAT != 0 && cfunc_argc == -1 - asm.incr_counter(:send_args_splat_cfunc_var_args) - return CantCompile - end - - if flags & C::VM_CALL_ARGS_SPLAT != 0 && flags & C::VM_CALL_ZSUPER != 0 - # zsuper methods are super calls without any arguments. - # They are also marked as splat, but don't actually have an array - # they pull arguments from, instead we need to change to call - # a different method with the current stack. - asm.incr_counter(:send_args_splat_cfunc_zuper) - return CantCompile; - end - - # In order to handle backwards compatibility between ruby 3 and 2 - # ruby2_keywords was introduced. It is called only on methods - # with splat and changes they way they handle them. - # We are just going to not compile these. - # https://docs.ruby-lang.org/en/3.2/Module.html#method-i-ruby2_keywords - if jit.iseq.body.param.flags.ruby2_keywords && flags & C::VM_CALL_ARGS_SPLAT != 0 - asm.incr_counter(:send_args_splat_cfunc_ruby2_keywords) - return CantCompile; - end - - kw_arg = calling.kwarg - kw_arg_num = if kw_arg.nil? - 0 - else - kw_arg.keyword_len - end - - if kw_arg_num != 0 && flags & C::VM_CALL_ARGS_SPLAT != 0 - asm.incr_counter(:send_cfunc_splat_with_kw) - return CantCompile - end - - if c_method_tracing_currently_enabled? - # Don't JIT if tracing c_call or c_return - asm.incr_counter(:send_cfunc_tracing) - return CantCompile - end - - # Delegate to codegen for C methods if we have it. - if kw_arg.nil? && flags & C::VM_CALL_OPT_SEND == 0 && flags & C::VM_CALL_ARGS_SPLAT == 0 && (cfunc_argc == -1 || argc == cfunc_argc) - known_cfunc_codegen = lookup_cfunc_codegen(cme.def) - if known_cfunc_codegen&.call(jit, ctx, asm, argc, known_recv_class) - # cfunc codegen generated code. Terminate the block so - # there isn't multiple calls in the same block. - jump_to_next_insn(jit, ctx, asm) - return EndBlock - end - end - - # Check for interrupts - jit_check_ints(jit, ctx, asm) - - # Stack overflow check - # #define CHECK_VM_STACK_OVERFLOW0(cfp, sp, margin) - # REG_CFP <= REG_SP + 4 * SIZEOF_VALUE + sizeof(rb_control_frame_t) - asm.comment('stack overflow check') - asm.lea(:rax, ctx.sp_opnd(C.VALUE.size * 4 + 2 * C.rb_control_frame_t.size)) - asm.cmp(CFP, :rax) - asm.jbe(counted_exit(side_exit(jit, ctx), :send_stackoverflow)) - - # Number of args which will be passed through to the callee - # This is adjusted by the kwargs being combined into a hash. - passed_argc = if kw_arg.nil? - argc - else - argc - kw_arg_num + 1 - end - - # If the argument count doesn't match - if cfunc_argc >= 0 && cfunc_argc != passed_argc && flags & C::VM_CALL_ARGS_SPLAT == 0 - asm.incr_counter(:send_cfunc_argc_mismatch) - return CantCompile - end - - # Don't JIT functions that need C stack arguments for now - if cfunc_argc >= 0 && passed_argc + 1 > C_ARGS.size - asm.incr_counter(:send_cfunc_toomany_args) - return CantCompile - end - - block_arg = flags & C::VM_CALL_ARGS_BLOCKARG != 0 - - # Guard block_arg_type - if guard_block_arg(jit, ctx, asm, calling) == CantCompile - return CantCompile - end - - if block_arg - ctx.stack_pop(1) - end - - # push_splat_args does stack manipulation so we can no longer side exit - if flags & C::VM_CALL_ARGS_SPLAT != 0 - assert_equal(true, cfunc_argc >= 0) - required_args = cfunc_argc - (argc - 1) - # + 1 because we pass self - if required_args + 1 >= C_ARGS.size - asm.incr_counter(:send_cfunc_toomany_args) - return CantCompile - end - - # We are going to assume that the splat fills - # all the remaining arguments. So the number of args - # should just equal the number of args the cfunc takes. - # In the generated code we test if this is true - # and if not side exit. - argc = cfunc_argc - passed_argc = argc - push_splat_args(required_args, jit, ctx, asm) - end - - # This is a .send call and we need to adjust the stack - if flags & C::VM_CALL_OPT_SEND != 0 - handle_opt_send_shift_stack(asm, argc, ctx, send_shift: calling.send_shift) - end - - # Points to the receiver operand on the stack - - # Store incremented PC into current control frame in case callee raises. - jit_save_pc(jit, asm) - - # Increment the stack pointer by 3 (in the callee) - # sp += 3 - - frame_type = C::VM_FRAME_MAGIC_CFUNC | C::VM_FRAME_FLAG_CFRAME | C::VM_ENV_FLAG_LOCAL - if kw_arg - frame_type |= C::VM_FRAME_FLAG_CFRAME_KW - end - - jit_push_frame(jit, ctx, asm, cme, flags, argc, frame_type, calling.block_handler) - - if kw_arg - # Build a hash from all kwargs passed - asm.comment('build_kwhash') - imemo_ci = calling.ci_addr - # we assume all callinfos with kwargs are on the GC heap - assert_equal(true, C.imemo_type_p(imemo_ci, C.imemo_callinfo)) - asm.mov(C_ARGS[0], imemo_ci) - asm.lea(C_ARGS[1], ctx.sp_opnd(0)) - asm.call(C.rjit_build_kwhash) - - # Replace the stack location at the start of kwargs with the new hash - stack_opnd = ctx.stack_opnd(argc - passed_argc) - asm.mov(stack_opnd, C_RET) - end - - # Copy SP because REG_SP will get overwritten - sp = :rax - asm.lea(sp, ctx.sp_opnd(0)) - - # Pop the C function arguments from the stack (in the caller) - ctx.stack_pop(argc + 1) - - # Write interpreter SP into CFP. - # Needed in case the callee yields to the block. - jit_save_sp(ctx, asm) - - # Non-variadic method - case cfunc_argc - in (0..) # Non-variadic method - # Copy the arguments from the stack to the C argument registers - # self is the 0th argument and is at index argc from the stack top - (0..passed_argc).each do |i| - asm.mov(C_ARGS[i], [sp, -(argc + 1 - i) * C.VALUE.size]) - end - in -1 # Variadic method: rb_f_puts(int argc, VALUE *argv, VALUE recv) - # The method gets a pointer to the first argument - # rb_f_puts(int argc, VALUE *argv, VALUE recv) - asm.mov(C_ARGS[0], passed_argc) - asm.lea(C_ARGS[1], [sp, -argc * C.VALUE.size]) # argv - asm.mov(C_ARGS[2], [sp, -(argc + 1) * C.VALUE.size]) # recv - end - - # Call the C function - # VALUE ret = (cfunc->func)(recv, argv[0], argv[1]); - # cfunc comes from compile-time cme->def, which we assume to be stable. - # Invalidation logic is in yjit_method_lookup_change() - asm.comment('call C function') - asm.mov(:rax, cfunc.func) - asm.call(:rax) # TODO: use rel32 if close enough - - # Record code position for TracePoint patching. See full_cfunc_return(). - Invariants.record_global_inval_patch(asm, full_cfunc_return) - - # Push the return value on the Ruby stack - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - - # Pop the stack frame (ec->cfp++) - # Instead of recalculating, we can reuse the previous CFP, which is stored in a callee-saved - # register - asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], CFP) - - # cfunc calls may corrupt types - ctx.clear_local_types - - # Note: the return block of jit_call_iseq has ctx->sp_offset == 1 - # which allows for sharing the same successor. - - # Jump (fall through) to the call continuation block - # We do this to end the current block after the call - assert_equal(1, ctx.sp_offset) - jump_to_next_insn(jit, ctx, asm) - EndBlock - end - - # vm_call_attrset - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_call_attrset(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd) - argc = calling.argc - flags = calling.flags - send_shift = calling.send_shift - - if flags & C::VM_CALL_ARGS_SPLAT != 0 - asm.incr_counter(:send_attrset_splat) - return CantCompile - end - if flags & C::VM_CALL_KWARG != 0 - asm.incr_counter(:send_attrset_kwarg) - return CantCompile - elsif argc != 1 || !C.RB_TYPE_P(comptime_recv, C::RUBY_T_OBJECT) - asm.incr_counter(:send_attrset_method) - return CantCompile - elsif c_method_tracing_currently_enabled? - # Can't generate code for firing c_call and c_return events - # See :attr-tracing: - asm.incr_counter(:send_c_tracingg) - return CantCompile - elsif flags & C::VM_CALL_ARGS_BLOCKARG != 0 - asm.incr_counter(:send_block_arg) - return CantCompile - end - - ivar_name = cme.def.body.attr.id - - # This is a .send call and we need to adjust the stack - if flags & C::VM_CALL_OPT_SEND != 0 - handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) - end - - # Save the PC and SP because the callee may allocate - # Note that this modifies REG_SP, which is why we do it first - jit_prepare_routine_call(jit, ctx, asm) - - # Get the operands from the stack - val_opnd = ctx.stack_pop(1) - recv_opnd = ctx.stack_pop(1) - - # Call rb_vm_set_ivar_id with the receiver, the ivar name, and the value - asm.mov(C_ARGS[0], recv_opnd) - asm.mov(C_ARGS[1], ivar_name) - asm.mov(C_ARGS[2], val_opnd) - asm.call(C.rb_vm_set_ivar_id) - - out_opnd = ctx.stack_push(Type::Unknown) - asm.mov(out_opnd, C_RET) - - KeepCompiling - end - - # vm_call_ivar (+ part of vm_call_method_each_type) - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_call_ivar(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd) - argc = calling.argc - flags = calling.flags - - if flags & C::VM_CALL_ARGS_SPLAT != 0 - asm.incr_counter(:send_ivar_splat) - return CantCompile - end - - if argc != 0 - asm.incr_counter(:send_arity) - return CantCompile - end - - # We don't support handle_opt_send_shift_stack for this yet. - if flags & C::VM_CALL_OPT_SEND != 0 - asm.incr_counter(:send_ivar_opt_send) - return CantCompile - end - - ivar_id = cme.def.body.attr.id - - # Not handling block_handler - if flags & C::VM_CALL_ARGS_BLOCKARG != 0 - asm.incr_counter(:send_block_arg) - return CantCompile - end - - jit_getivar(jit, ctx, asm, comptime_recv, ivar_id, recv_opnd, StackOpnd[0]) - end - - # vm_call_bmethod - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_call_bmethod(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) - proc_addr = cme.def.body.bmethod.proc - - proc_t = C.rb_yjit_get_proc_ptr(proc_addr) - proc_block = proc_t.block - - if proc_block.type != C.block_type_iseq - asm.incr_counter(:send_bmethod_not_iseq) - return CantCompile - end - - capture = proc_block.as.captured - iseq = capture.code.iseq - - # TODO: implement this - # Optimize for single ractor mode and avoid runtime check for - # "defined with an un-shareable Proc in a different Ractor" - # if !assume_single_ractor_mode(jit, ocb) - # return CantCompile; - # end - - # Passing a block to a block needs logic different from passing - # a block to a method and sometimes requires allocation. Bail for now. - if calling.block_handler != C::VM_BLOCK_HANDLER_NONE - asm.incr_counter(:send_bmethod_blockarg) - return CantCompile - end - - jit_call_iseq( - jit, ctx, asm, cme, calling, iseq, - frame_type: C::VM_FRAME_MAGIC_BLOCK | C::VM_FRAME_FLAG_BMETHOD | C::VM_FRAME_FLAG_LAMBDA, - prev_ep: capture.ep, - ) - end - - # vm_call_alias - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_call_alias(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) - cme = C.rb_aliased_callable_method_entry(cme) - jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) - end - - # vm_call_optimized - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_call_optimized(jit, ctx, asm, cme, calling, known_recv_class) - if calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 - # Not working yet - asm.incr_counter(:send_block_arg) - return CantCompile - end - - case cme.def.body.optimized.type - in C::OPTIMIZED_METHOD_TYPE_SEND - jit_call_opt_send(jit, ctx, asm, cme, calling, known_recv_class) - in C::OPTIMIZED_METHOD_TYPE_CALL - jit_call_opt_call(jit, ctx, asm, cme, calling.flags, calling.argc, calling.block_handler, known_recv_class, send_shift: calling.send_shift) - in C::OPTIMIZED_METHOD_TYPE_BLOCK_CALL - asm.incr_counter(:send_optimized_block_call) - return CantCompile - in C::OPTIMIZED_METHOD_TYPE_STRUCT_AREF - jit_call_opt_struct_aref(jit, ctx, asm, cme, calling.flags, calling.argc, calling.block_handler, known_recv_class, send_shift: calling.send_shift) - in C::OPTIMIZED_METHOD_TYPE_STRUCT_ASET - asm.incr_counter(:send_optimized_struct_aset) - return CantCompile - end - end - - # vm_call_opt_send - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_call_opt_send(jit, ctx, asm, cme, calling, known_recv_class) - if jit_caller_setup_arg(jit, ctx, asm, calling.flags) == CantCompile - return CantCompile - end - - if calling.argc == 0 - asm.incr_counter(:send_optimized_send_no_args) - return CantCompile - end - - calling.argc -= 1 - # We aren't handling `send(:send, ...)` yet. This might work, but not tested yet. - if calling.send_shift > 0 - asm.incr_counter(:send_optimized_send_send) - return CantCompile - end - # Lazily handle stack shift in handle_opt_send_shift_stack - calling.send_shift += 1 - - jit_call_symbol(jit, ctx, asm, cme, calling, known_recv_class, C::VM_CALL_FCALL) - end - - # vm_call_opt_call - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_call_opt_call(jit, ctx, asm, cme, flags, argc, block_handler, known_recv_class, send_shift:) - if block_handler != C::VM_BLOCK_HANDLER_NONE - asm.incr_counter(:send_optimized_call_block) - return CantCompile - end - - if flags & C::VM_CALL_KWARG != 0 - asm.incr_counter(:send_optimized_call_kwarg) - return CantCompile - end - - if flags & C::VM_CALL_ARGS_SPLAT != 0 - asm.incr_counter(:send_optimized_call_splat) - return CantCompile - end - - # TODO: implement this - # Optimize for single ractor mode and avoid runtime check for - # "defined with an un-shareable Proc in a different Ractor" - # if !assume_single_ractor_mode(jit, ocb) - # return CantCompile - # end - - # If this is a .send call we need to adjust the stack - if flags & C::VM_CALL_OPT_SEND != 0 - handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) - end - - # About to reset the SP, need to load this here - recv_idx = argc # blockarg is not supported. send_shift is already handled. - asm.mov(:rcx, ctx.stack_opnd(recv_idx)) # recv - - # Save the PC and SP because the callee can make Ruby calls - jit_prepare_routine_call(jit, ctx, asm) # NOTE: clobbers rax - - asm.lea(:rax, ctx.sp_opnd(0)) # sp - - kw_splat = flags & C::VM_CALL_KW_SPLAT - - asm.mov(C_ARGS[0], :rcx) - asm.mov(C_ARGS[1], EC) - asm.mov(C_ARGS[2], argc) - asm.lea(C_ARGS[3], [:rax, -argc * C.VALUE.size]) # stack_argument_pointer. NOTE: C_ARGS[3] is rcx - asm.mov(C_ARGS[4], kw_splat) - asm.mov(C_ARGS[5], C::VM_BLOCK_HANDLER_NONE) - asm.call(C.rjit_optimized_call) - - ctx.stack_pop(argc + 1) - - stack_ret = ctx.stack_push(Type::Unknown) - asm.mov(stack_ret, C_RET) - return KeepCompiling - end - - # vm_call_opt_struct_aref - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_call_opt_struct_aref(jit, ctx, asm, cme, flags, argc, block_handler, known_recv_class, send_shift:) - if argc != 0 - asm.incr_counter(:send_optimized_struct_aref_error) - return CantCompile - end - - if c_method_tracing_currently_enabled? - # Don't JIT if tracing c_call or c_return - asm.incr_counter(:send_cfunc_tracing) - return CantCompile - end - - off = cme.def.body.optimized.index - - recv_idx = argc # blockarg is not supported - recv_idx += send_shift - comptime_recv = jit.peek_at_stack(recv_idx) - - # This is a .send call and we need to adjust the stack - if flags & C::VM_CALL_OPT_SEND != 0 - handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) - end - - # All structs from the same Struct class should have the same - # length. So if our comptime_recv is embedded all runtime - # structs of the same class should be as well, and the same is - # true of the converse. - embedded = C::FL_TEST_RAW(comptime_recv, C::RSTRUCT_EMBED_LEN_MASK) - - asm.comment('struct aref') - asm.mov(:rax, ctx.stack_pop(1)) # recv - - if embedded - asm.mov(:rax, [:rax, C.RStruct.offsetof(:as, :ary) + (C.VALUE.size * off)]) - else - asm.mov(:rax, [:rax, C.RStruct.offsetof(:as, :heap, :ptr)]) - asm.mov(:rax, [:rax, C.VALUE.size * off]) - end - - ret = ctx.stack_push(Type::Unknown) - asm.mov(ret, :rax) - - jump_to_next_insn(jit, ctx, asm) - EndBlock - end - - # vm_call_opt_send (lazy part) - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) - # We don't support `send(:send, ...)` for now. - assert_equal(1, send_shift) - - asm.comment('shift stack') - (0...argc).reverse_each do |i| - opnd = ctx.stack_opnd(i) - opnd2 = ctx.stack_opnd(i + 1) - asm.mov(:rax, opnd) - asm.mov(opnd2, :rax) - end - - ctx.shift_stack(argc) - end - - # vm_call_symbol - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_call_symbol(jit, ctx, asm, cme, calling, known_recv_class, flags) - flags |= C::VM_CALL_OPT_SEND | (calling.kw_splat ? C::VM_CALL_KW_SPLAT : 0) - - comptime_symbol = jit.peek_at_stack(calling.argc) - if comptime_symbol.class != String && !static_symbol?(comptime_symbol) - asm.incr_counter(:send_optimized_send_not_sym_or_str) - return CantCompile - end - - mid = C.get_symbol_id(comptime_symbol) - if mid == 0 - asm.incr_counter(:send_optimized_send_null_mid) - return CantCompile - end - - asm.comment("Guard #{comptime_symbol.inspect} is on stack") - class_changed_exit = counted_exit(side_exit(jit, ctx), :send_optimized_send_mid_class_changed) - jit_guard_known_klass( - jit, ctx, asm, C.rb_class_of(comptime_symbol), ctx.stack_opnd(calling.argc), - StackOpnd[calling.argc], comptime_symbol, class_changed_exit, - ) - asm.mov(C_ARGS[0], ctx.stack_opnd(calling.argc)) - asm.call(C.rb_get_symbol_id) - asm.cmp(C_RET, mid) - id_changed_exit = counted_exit(side_exit(jit, ctx), :send_optimized_send_mid_id_changed) - jit_chain_guard(:jne, jit, ctx, asm, id_changed_exit) - - # rb_callable_method_entry_with_refinements - calling.flags = flags - cme, _ = jit_search_method(jit, ctx, asm, mid, calling) - if cme == CantCompile - return CantCompile - end - - if flags & C::VM_CALL_FCALL != 0 - return jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class) - end - - raise NotImplementedError # unreachable for now - end - - # vm_push_frame - # - # Frame structure: - # | args | locals | cme/cref | block_handler/prev EP | frame type (EP here) | stack bottom (SP here) - # - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_push_frame(jit, ctx, asm, cme, flags, argc, frame_type, block_handler, iseq: nil, local_size: 0, stack_max: 0, prev_ep: nil, doing_kw_call: nil) - # Save caller SP and PC before pushing a callee frame for backtrace and side exits - asm.comment('save SP to caller CFP') - recv_idx = argc # blockarg is already popped - recv_idx += (block_handler == :captured) ? 0 : 1 # receiver is not on stack when captured->self is used - if iseq - # Skip setting this to SP register. This cfp->sp will be copied to SP on leave insn. - asm.lea(:rax, ctx.sp_opnd(C.VALUE.size * -recv_idx)) # Pop receiver and arguments to prepare for side exits - asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], :rax) - else - asm.lea(SP, ctx.sp_opnd(C.VALUE.size * -recv_idx)) - asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], SP) - ctx.sp_offset = recv_idx - end - jit_save_pc(jit, asm, comment: 'save PC to caller CFP') - - sp_offset = ctx.sp_offset + 3 + local_size + (doing_kw_call ? 1 : 0) # callee_sp - local_size.times do |i| - asm.comment('set local variables') if i == 0 - local_index = sp_offset + i - local_size - 3 - asm.mov([SP, C.VALUE.size * local_index], Qnil) - end - - asm.comment('set up EP with managing data') - ep_offset = sp_offset - 1 - # ep[-2]: cref_or_me - asm.mov(:rax, cme.to_i) - asm.mov([SP, C.VALUE.size * (ep_offset - 2)], :rax) - # ep[-1]: block handler or prev env ptr (specval) - if prev_ep - asm.mov(:rax, prev_ep.to_i | 1) # tagged prev ep - asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax) - elsif block_handler == :captured - # Set captured->ep, saving captured in :rcx for captured->self - ep_reg = :rcx - jit_get_lep(jit, asm, reg: ep_reg) - asm.mov(:rcx, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler - asm.and(:rcx, ~0x3) # captured - asm.mov(:rax, [:rcx, C.VALUE.size]) # captured->ep - asm.or(:rax, 0x1) # GC_GUARDED_PTR - asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax) - elsif block_handler == C::VM_BLOCK_HANDLER_NONE - asm.mov([SP, C.VALUE.size * (ep_offset - 1)], C::VM_BLOCK_HANDLER_NONE) - elsif block_handler == C.rb_block_param_proxy - # vm_caller_setup_arg_block: block_code == rb_block_param_proxy - jit_get_lep(jit, asm, reg: :rax) # VM_CF_BLOCK_HANDLER: VM_CF_LEP - asm.mov(:rax, [:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # VM_CF_BLOCK_HANDLER: VM_ENV_BLOCK_HANDLER - asm.mov([CFP, C.rb_control_frame_t.offsetof(:block_code)], :rax) # reg_cfp->block_code = handler - asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax) # return handler; - else # assume blockiseq - asm.mov(:rax, block_handler) - asm.mov([CFP, C.rb_control_frame_t.offsetof(:block_code)], :rax) - asm.lea(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) # VM_CFP_TO_CAPTURED_BLOCK - asm.or(:rax, 1) # VM_BH_FROM_ISEQ_BLOCK - asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax) - end - # ep[-0]: ENV_FLAGS - asm.mov([SP, C.VALUE.size * (ep_offset - 0)], frame_type) - - asm.comment('set up new frame') - cfp_offset = -C.rb_control_frame_t.size # callee CFP - # For ISEQ, JIT code will set it as needed. However, C func needs 0 there for svar frame detection. - if iseq.nil? - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:pc)], 0) - end - asm.mov(:rax, iseq.to_i) - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:iseq)], :rax) - if block_handler == :captured - asm.mov(:rax, [:rcx]) # captured->self - else - self_index = ctx.sp_offset - (1 + argc) # blockarg has been popped - asm.mov(:rax, [SP, C.VALUE.size * self_index]) - end - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:self)], :rax) - asm.lea(:rax, [SP, C.VALUE.size * ep_offset]) - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:ep)], :rax) - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:block_code)], 0) - # Update SP register only for ISEQ calls. SP-relative operations should be done above this. - sp_reg = iseq ? SP : :rax - asm.lea(sp_reg, [SP, C.VALUE.size * sp_offset]) - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:sp)], sp_reg) - - # cfp->jit_return is used only for ISEQs - if iseq - # The callee might change locals through Kernel#binding and other means. - ctx.clear_local_types - - # Stub cfp->jit_return - return_ctx = ctx.dup - return_ctx.stack_pop(argc + ((block_handler == :captured) ? 0 : 1)) # Pop args and receiver. blockarg has been popped - return_ctx.stack_push(Type::Unknown) # push callee's return value - return_ctx.sp_offset = 1 # SP is in the position after popping a receiver and arguments - return_ctx.chain_depth = 0 - branch_stub = BranchStub.new( - iseq: jit.iseq, - shape: Default, - target0: BranchTarget.new(ctx: return_ctx, pc: jit.pc + jit.insn.len * C.VALUE.size), - ) - branch_stub.target0.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(return_ctx, ocb_asm, branch_stub, true) - @ocb.write(ocb_asm) - end - branch_stub.compile = compile_jit_return(branch_stub, cfp_offset:) - branch_stub.compile.call(asm) - end - - asm.comment('switch to callee CFP') - # Update CFP register only for ISEQ calls - cfp_reg = iseq ? CFP : :rax - asm.lea(cfp_reg, [CFP, cfp_offset]) - asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], cfp_reg) - end - - def compile_jit_return(branch_stub, cfp_offset:) # Proc escapes arguments in memory - proc do |branch_asm| - branch_asm.comment('set jit_return to callee CFP') - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.mov(:rax, branch_stub.target0.address) - branch_asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:jit_return)], :rax) - end - end - end - end - - # CALLER_SETUP_ARG: Return CantCompile if not supported - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def jit_caller_setup_arg(jit, ctx, asm, flags) - if flags & C::VM_CALL_ARGS_SPLAT != 0 && flags & C::VM_CALL_KW_SPLAT != 0 - asm.incr_counter(:send_args_splat_kw_splat) - return CantCompile - elsif flags & C::VM_CALL_ARGS_SPLAT != 0 - # splat is not supported in this path - asm.incr_counter(:send_args_splat) - return CantCompile - elsif flags & C::VM_CALL_KW_SPLAT != 0 - asm.incr_counter(:send_args_kw_splat) - return CantCompile - elsif flags & C::VM_CALL_KWARG != 0 - asm.incr_counter(:send_kwarg) - return CantCompile - end - end - - # Pushes arguments from an array to the stack. Differs from push splat because - # the array can have items left over. - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def move_rest_args_to_stack(array, num_args, jit, ctx, asm) - side_exit = side_exit(jit, ctx) - - asm.comment('move_rest_args_to_stack') - - # array is :rax - array_len_opnd = :rcx - jit_array_len(asm, array, array_len_opnd) - - asm.comment('Side exit if length is less than required') - asm.cmp(array_len_opnd, num_args) - asm.jl(counted_exit(side_exit, :send_iseq_has_rest_and_splat_not_equal)) - - asm.comment('Push arguments from array') - - # Load the address of the embedded array - # (struct RArray *)(obj)->as.ary - array_reg = array - - # Conditionally load the address of the heap array - # (struct RArray *)(obj)->as.heap.ptr - flags_opnd = [array_reg, C.RBasic.offsetof(:flags)] - asm.test(flags_opnd, C::RARRAY_EMBED_FLAG) - heap_ptr_opnd = [array_reg, C.RArray.offsetof(:as, :heap, :ptr)] - # Load the address of the embedded array - # (struct RArray *)(obj)->as.ary - ary_opnd = :rdx # NOTE: array :rax is used after move_rest_args_to_stack too - asm.lea(:rcx, [array_reg, C.RArray.offsetof(:as, :ary)]) - asm.mov(ary_opnd, heap_ptr_opnd) - asm.cmovnz(ary_opnd, :rcx) - - num_args.times do |i| - top = ctx.stack_push(Type::Unknown) - asm.mov(:rcx, [ary_opnd, i * C.VALUE.size]) - asm.mov(top, :rcx) - end - end - - # vm_caller_setup_arg_splat (+ CALLER_SETUP_ARG): - # Pushes arguments from an array to the stack that are passed with a splat (i.e. *args). - # It optimistically compiles to a static size that is the exact number of arguments needed for the function. - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def push_splat_args(required_args, jit, ctx, asm) - side_exit = side_exit(jit, ctx) - - asm.comment('push_splat_args') - - array_opnd = ctx.stack_opnd(0) - array_stack_opnd = StackOpnd[0] - array_reg = :rax - asm.mov(array_reg, array_opnd) - - guard_object_is_array(jit, ctx, asm, array_reg, :rcx, array_stack_opnd, :send_args_splat_not_array) - - array_len_opnd = :rcx - jit_array_len(asm, array_reg, array_len_opnd) - - asm.comment('Side exit if length is not equal to remaining args') - asm.cmp(array_len_opnd, required_args) - asm.jne(counted_exit(side_exit, :send_args_splat_length_not_equal)) - - asm.comment('Check last argument is not ruby2keyword hash') - - ary_opnd = :rcx - jit_array_ptr(asm, array_reg, ary_opnd) # clobbers array_reg - - last_array_value = :rax - asm.mov(last_array_value, [ary_opnd, (required_args - 1) * C.VALUE.size]) - - ruby2_exit = counted_exit(side_exit, :send_args_splat_ruby2_hash); - guard_object_is_not_ruby2_keyword_hash(asm, last_array_value, :rcx, ruby2_exit) # clobbers :rax - - asm.comment('Push arguments from array') - array_opnd = ctx.stack_pop(1) - - if required_args > 0 - # Load the address of the embedded array - # (struct RArray *)(obj)->as.ary - array_reg = :rax - asm.mov(array_reg, array_opnd) - - # Conditionally load the address of the heap array - # (struct RArray *)(obj)->as.heap.ptr - flags_opnd = [array_reg, C.RBasic.offsetof(:flags)] - asm.test(flags_opnd, C::RARRAY_EMBED_FLAG) - heap_ptr_opnd = [array_reg, C.RArray.offsetof(:as, :heap, :ptr)] - # Load the address of the embedded array - # (struct RArray *)(obj)->as.ary - asm.lea(:rcx, [array_reg, C.RArray.offsetof(:as, :ary)]) - asm.mov(:rax, heap_ptr_opnd) - asm.cmovnz(:rax, :rcx) - ary_opnd = :rax - - (0...required_args).each do |i| - top = ctx.stack_push(Type::Unknown) - asm.mov(:rcx, [ary_opnd, i * C.VALUE.size]) - asm.mov(top, :rcx) - end - - asm.comment('end push_each') - end - end - - # Generate RARRAY_LEN. For array_opnd, use Opnd::Reg to reduce memory access, - # and use Opnd::Mem to save registers. - def jit_array_len(asm, array_reg, len_reg) - asm.comment('get array length for embedded or heap') - - # Pull out the embed flag to check if it's an embedded array. - asm.mov(len_reg, [array_reg, C.RBasic.offsetof(:flags)]) - - # Get the length of the array - asm.and(len_reg, C::RARRAY_EMBED_LEN_MASK) - asm.sar(len_reg, C::RARRAY_EMBED_LEN_SHIFT) - - # Conditionally move the length of the heap array - asm.test([array_reg, C.RBasic.offsetof(:flags)], C::RARRAY_EMBED_FLAG) - - # Select the array length value - asm.cmovz(len_reg, [array_reg, C.RArray.offsetof(:as, :heap, :len)]) - end - - # Generate RARRAY_CONST_PTR (part of RARRAY_AREF) - def jit_array_ptr(asm, array_reg, ary_opnd) # clobbers array_reg - asm.comment('get array pointer for embedded or heap') - - flags_opnd = [array_reg, C.RBasic.offsetof(:flags)] - asm.test(flags_opnd, C::RARRAY_EMBED_FLAG) - # Load the address of the embedded array - # (struct RArray *)(obj)->as.ary - asm.mov(ary_opnd, [array_reg, C.RArray.offsetof(:as, :heap, :ptr)]) - asm.lea(array_reg, [array_reg, C.RArray.offsetof(:as, :ary)]) # clobbers array_reg - asm.cmovnz(ary_opnd, array_reg) - end - - def assert(cond) - assert_equal(cond, true) - end - - def assert_equal(left, right) - if left != right - raise "'#{left.inspect}' was not '#{right.inspect}'" - end - end - - def fixnum?(obj) - (C.to_value(obj) & C::RUBY_FIXNUM_FLAG) == C::RUBY_FIXNUM_FLAG - end - - def flonum?(obj) - (C.to_value(obj) & C::RUBY_FLONUM_MASK) == C::RUBY_FLONUM_FLAG - end - - def symbol?(obj) - static_symbol?(obj) || dynamic_symbol?(obj) - end - - def static_symbol?(obj) - (C.to_value(obj) & 0xff) == C::RUBY_SYMBOL_FLAG - end - - def dynamic_symbol?(obj) - return false if C::SPECIAL_CONST_P(obj) - C.RB_TYPE_P(obj, C::RUBY_T_SYMBOL) - end - - def shape_too_complex?(obj) - C.rb_shape_get_shape_id(obj) == C::OBJ_TOO_COMPLEX_SHAPE_ID - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - # @param asm [RubyVM::RJIT::Assembler] - def defer_compilation(jit, ctx, asm) - # Make a stub to compile the current insn - if ctx.chain_depth != 0 - raise "double defer!" - end - ctx.chain_depth += 1 - jit_direct_jump(jit.iseq, jit.pc, ctx, asm, comment: 'defer_compilation') - end - - def jit_direct_jump(iseq, pc, ctx, asm, comment: 'jit_direct_jump') - branch_stub = BranchStub.new( - iseq:, - shape: Default, - target0: BranchTarget.new(ctx:, pc:), - ) - branch_stub.target0.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true) - @ocb.write(ocb_asm) - end - branch_stub.compile = compile_jit_direct_jump(branch_stub, comment:) - branch_stub.compile.call(asm) - end - - def compile_jit_direct_jump(branch_stub, comment:) # Proc escapes arguments in memory - proc do |branch_asm| - branch_asm.comment(comment) - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.jmp(branch_stub.target0.address) - in Next0 - # Just write the block without a jump - end - end - end - end - - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - def side_exit(jit, ctx) - # We use the latest ctx.sp_offset to generate a side exit to tolerate sp_offset changes by jit_save_sp. - # However, we want to simulate an old stack_size when we take a side exit. We do that by adjusting the - # sp_offset because gen_outlined_exit uses ctx.sp_offset to move SP. - ctx = ctx.with_stack_size(jit.stack_size_for_pc) - - jit.side_exit_for_pc[ctx.sp_offset] ||= Assembler.new.then do |asm| - @exit_compiler.compile_side_exit(jit.pc, ctx, asm) - @ocb.write(asm) - end - end - - def counted_exit(side_exit, name) - asm = Assembler.new - asm.incr_counter(name) - asm.jmp(side_exit) - @ocb.write(asm) - end - - def def_iseq_ptr(cme_def) - C.rb_iseq_check(cme_def.body.iseq.iseqptr) - end - - def to_value(obj) - GC_REFS << obj - C.to_value(obj) - end - - def full_cfunc_return - @full_cfunc_return ||= Assembler.new.then do |asm| - @exit_compiler.compile_full_cfunc_return(asm) - @ocb.write(asm) - end - end - - def c_method_tracing_currently_enabled? - C.rb_rjit_global_events & (C::RUBY_EVENT_C_CALL | C::RUBY_EVENT_C_RETURN) != 0 - end - - # Return a builtin function if a given iseq consists of only that builtin function - def builtin_function(iseq) - opt_invokebuiltin_delegate_leave = INSNS.values.find { |i| i.name == :opt_invokebuiltin_delegate_leave } - leave = INSNS.values.find { |i| i.name == :leave } - if iseq.body.iseq_size == opt_invokebuiltin_delegate_leave.len + leave.len && - C.rb_vm_insn_decode(iseq.body.iseq_encoded[0]) == opt_invokebuiltin_delegate_leave.bin && - C.rb_vm_insn_decode(iseq.body.iseq_encoded[opt_invokebuiltin_delegate_leave.len]) == leave.bin - C.rb_builtin_function.new(iseq.body.iseq_encoded[1]) - end - end - - def build_calling(ci:, block_handler:) - CallingInfo.new( - argc: C.vm_ci_argc(ci), - flags: C.vm_ci_flag(ci), - kwarg: C.vm_ci_kwarg(ci), - ci_addr: ci.to_i, - send_shift: 0, - block_handler:, - ) - end - end -end |