diff options
Diffstat (limited to 'lib/ruby_vm/rjit/compiler.rb')
| -rw-r--r-- | lib/ruby_vm/rjit/compiler.rb | 518 |
1 files changed, 0 insertions, 518 deletions
diff --git a/lib/ruby_vm/rjit/compiler.rb b/lib/ruby_vm/rjit/compiler.rb deleted file mode 100644 index e5c3adf0ec..0000000000 --- a/lib/ruby_vm/rjit/compiler.rb +++ /dev/null @@ -1,518 +0,0 @@ -require 'ruby_vm/rjit/assembler' -require 'ruby_vm/rjit/block' -require 'ruby_vm/rjit/branch_stub' -require 'ruby_vm/rjit/code_block' -require 'ruby_vm/rjit/context' -require 'ruby_vm/rjit/entry_stub' -require 'ruby_vm/rjit/exit_compiler' -require 'ruby_vm/rjit/insn_compiler' -require 'ruby_vm/rjit/instruction' -require 'ruby_vm/rjit/invariants' -require 'ruby_vm/rjit/jit_state' -require 'ruby_vm/rjit/type' - -module RubyVM::RJIT - # Compilation status - KeepCompiling = :KeepCompiling - CantCompile = :CantCompile - EndBlock = :EndBlock - - # Ruby constants - Qtrue = Fiddle::Qtrue - Qfalse = Fiddle::Qfalse - Qnil = Fiddle::Qnil - Qundef = Fiddle::Qundef - - # Callee-saved registers - # TODO: support using r12/r13 here - EC = :r14 - CFP = :r15 - SP = :rbx - - # Scratch registers: rax, rcx, rdx - - # Mark objects in this Array during GC - GC_REFS = [] - - # Maximum number of versions per block - # 1 means always create generic versions - MAX_VERSIONS = 4 - - class Compiler - attr_accessor :write_pos - - def self.decode_insn(encoded) - INSNS.fetch(C.rb_vm_insn_decode(encoded)) - end - - def initialize - mem_size = C.rjit_opts.exec_mem_size * 1024 * 1024 - mem_block = C.mmap(mem_size) - @cb = CodeBlock.new(mem_block: mem_block, mem_size: mem_size / 2) - @ocb = CodeBlock.new(mem_block: mem_block + mem_size / 2, mem_size: mem_size / 2, outlined: true) - @exit_compiler = ExitCompiler.new - @insn_compiler = InsnCompiler.new(@cb, @ocb, @exit_compiler) - Invariants.initialize(@cb, @ocb, self, @exit_compiler) - end - - # Compile an ISEQ from its entry point. - # @param iseq `RubyVM::RJIT::CPointer::Struct_rb_iseq_t` - # @param cfp `RubyVM::RJIT::CPointer::Struct_rb_control_frame_t` - def compile(iseq, cfp) - return unless supported_platform? - pc = cfp.pc.to_i - jit = JITState.new(iseq:, cfp:) - asm = Assembler.new - compile_prologue(asm, iseq, pc) - compile_block(asm, jit:, pc:) - iseq.body.jit_entry = @cb.write(asm) - rescue Exception => e - STDERR.puts "#{e.class}: #{e.message}" - STDERR.puts e.backtrace - exit 1 - end - - # Compile an entry. - # @param entry [RubyVM::RJIT::EntryStub] - def entry_stub_hit(entry_stub, cfp) - # Compile a new entry guard as a next entry - pc = cfp.pc.to_i - next_entry = Assembler.new.then do |asm| - compile_entry_chain_guard(asm, cfp.iseq, pc) - @cb.write(asm) - end - - # Try to find an existing compiled version of this block - ctx = Context.new - block = find_block(cfp.iseq, pc, ctx) - if block - # If an existing block is found, generate a jump to the block. - asm = Assembler.new - asm.jmp(block.start_addr) - @cb.write(asm) - else - # If this block hasn't yet been compiled, generate blocks after the entry guard. - asm = Assembler.new - jit = JITState.new(iseq: cfp.iseq, cfp:) - compile_block(asm, jit:, pc:, ctx:) - @cb.write(asm) - - block = jit.block - end - - # Regenerate the previous entry - @cb.with_write_addr(entry_stub.start_addr) do - # The last instruction of compile_entry_chain_guard is jne - asm = Assembler.new - asm.jne(next_entry) - @cb.write(asm) - end - - return block.start_addr - rescue Exception => e - STDERR.puts e.full_message - exit 1 - end - - # Compile a branch stub. - # @param branch_stub [RubyVM::RJIT::BranchStub] - # @param cfp `RubyVM::RJIT::CPointer::Struct_rb_control_frame_t` - # @param target0_p [TrueClass,FalseClass] - # @return [Integer] The starting address of the compiled branch stub - def branch_stub_hit(branch_stub, cfp, target0_p) - # Update cfp->pc for `jit.at_current_insn?` - target = target0_p ? branch_stub.target0 : branch_stub.target1 - cfp.pc = target.pc - - # Reuse an existing block if it already exists - block = find_block(branch_stub.iseq, target.pc, target.ctx) - - # If the branch stub's jump is the last code, allow overwriting part of - # the old branch code with the new block code. - fallthrough = block.nil? && @cb.write_addr == branch_stub.end_addr - if fallthrough - # If the branch stub's jump is the last code, allow overwriting part of - # the old branch code with the new block code. - @cb.set_write_addr(branch_stub.start_addr) - branch_stub.shape = target0_p ? Next0 : Next1 - Assembler.new.tap do |branch_asm| - branch_stub.compile.call(branch_asm) - @cb.write(branch_asm) - end - end - - # Reuse or generate a block - if block - target.address = block.start_addr - else - jit = JITState.new(iseq: branch_stub.iseq, cfp:) - target.address = Assembler.new.then do |asm| - compile_block(asm, jit:, pc: target.pc, ctx: target.ctx.dup) - @cb.write(asm) - end - block = jit.block - end - block.incoming << branch_stub # prepare for invalidate_block - - # Re-generate the branch code for non-fallthrough cases - unless fallthrough - @cb.with_write_addr(branch_stub.start_addr) do - branch_asm = Assembler.new - branch_stub.compile.call(branch_asm) - @cb.write(branch_asm) - end - end - - return target.address - rescue Exception => e - STDERR.puts e.full_message - exit 1 - end - - # @param iseq `RubyVM::RJIT::CPointer::Struct_rb_iseq_t` - # @param pc [Integer] - def invalidate_blocks(iseq, pc) - list_blocks(iseq, pc).each do |block| - invalidate_block(block) - end - - # If they were the ISEQ's first blocks, re-compile RJIT entry as well - if iseq.body.iseq_encoded.to_i == pc - iseq.body.jit_entry = 0 - iseq.body.jit_entry_calls = 0 - end - end - - def invalidate_block(block) - iseq = block.iseq - # Avoid touching GCed ISEQs. We assume it won't be re-entered. - return unless C.imemo_type_p(iseq, C.imemo_iseq) - - # Remove this block from the version array - remove_block(iseq, block) - - # Invalidate the block with entry exit - unless block.invalidated - @cb.with_write_addr(block.start_addr) do - asm = Assembler.new - asm.comment('invalidate_block') - asm.jmp(block.entry_exit) - @cb.write(asm) - end - block.invalidated = true - end - - # Re-stub incoming branches - block.incoming.each do |branch_stub| - target = [branch_stub.target0, branch_stub.target1].compact.find do |target| - target.pc == block.pc && target.ctx == block.ctx - end - next if target.nil? - # TODO: Could target.address be a stub address? Is invalidation not needed in that case? - - # If the target being re-generated is currently a fallthrough block, - # the fallthrough code must be rewritten with a jump to the stub. - if target.address == branch_stub.end_addr - branch_stub.shape = Default - end - - target.address = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_branch_stub(block.ctx, ocb_asm, branch_stub, target == branch_stub.target0) - @ocb.write(ocb_asm) - end - @cb.with_write_addr(branch_stub.start_addr) do - branch_asm = Assembler.new - branch_stub.compile.call(branch_asm) - @cb.write(branch_asm) - end - end - end - - private - - # Callee-saved: rbx, rsp, rbp, r12, r13, r14, r15 - # Caller-saved: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11 - # - # @param asm [RubyVM::RJIT::Assembler] - def compile_prologue(asm, iseq, pc) - asm.comment('RJIT entry point') - - # Save callee-saved registers used by JITed code - asm.push(CFP) - asm.push(EC) - asm.push(SP) - - # Move arguments EC and CFP to dedicated registers - asm.mov(EC, :rdi) - asm.mov(CFP, :rsi) - - # Load sp to a dedicated register - asm.mov(SP, [CFP, C.rb_control_frame_t.offsetof(:sp)]) # rbx = cfp->sp - - # Setup cfp->jit_return - asm.mov(:rax, leave_exit) - asm.mov([CFP, C.rb_control_frame_t.offsetof(:jit_return)], :rax) - - # We're compiling iseqs that we *expect* to start at `insn_idx`. But in - # the case of optional parameters, the interpreter can set the pc to a - # different location depending on the optional parameters. If an iseq - # has optional parameters, we'll add a runtime check that the PC we've - # compiled for is the same PC that the interpreter wants us to run with. - # If they don't match, then we'll take a side exit. - if iseq.body.param.flags.has_opt - compile_entry_chain_guard(asm, iseq, pc) - end - end - - def compile_entry_chain_guard(asm, iseq, pc) - entry_stub = EntryStub.new - stub_addr = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_entry_stub(ocb_asm, entry_stub) - @ocb.write(ocb_asm) - end - - asm.comment('guard expected PC') - asm.mov(:rax, pc) - asm.cmp([CFP, C.rb_control_frame_t.offsetof(:pc)], :rax) - - asm.stub(entry_stub) do - asm.jne(stub_addr) - end - end - - # @param asm [RubyVM::RJIT::Assembler] - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - def compile_block(asm, jit:, pc:, ctx: Context.new) - # Mark the block start address and prepare an exit code storage - ctx = limit_block_versions(jit.iseq, pc, ctx) - block = Block.new(iseq: jit.iseq, pc:, ctx: ctx.dup) - jit.block = block - asm.block(block) - - iseq = jit.iseq - asm.comment("Block: #{iseq.body.location.label}@#{C.rb_iseq_path(iseq)}:#{iseq_lineno(iseq, pc)}") - - # Compile each insn - index = (pc - iseq.body.iseq_encoded.to_i) / C.VALUE.size - while index < iseq.body.iseq_size - # Set the current instruction - insn = self.class.decode_insn(iseq.body.iseq_encoded[index]) - jit.pc = (iseq.body.iseq_encoded + index).to_i - jit.stack_size_for_pc = ctx.stack_size - jit.side_exit_for_pc.clear - - # If previous instruction requested to record the boundary - if jit.record_boundary_patch_point - # Generate an exit to this instruction and record it - exit_pos = Assembler.new.then do |ocb_asm| - @exit_compiler.compile_side_exit(jit.pc, ctx, ocb_asm) - @ocb.write(ocb_asm) - end - Invariants.record_global_inval_patch(asm, exit_pos) - jit.record_boundary_patch_point = false - end - - # In debug mode, verify our existing assumption - if C.rjit_opts.verify_ctx && jit.at_current_insn? - verify_ctx(jit, ctx) - end - - case status = @insn_compiler.compile(jit, ctx, asm, insn) - when KeepCompiling - # For now, reset the chain depth after each instruction as only the - # first instruction in the block can concern itself with the depth. - ctx.chain_depth = 0 - - index += insn.len - when EndBlock - # TODO: pad nops if entry exit exists (not needed for x86_64?) - break - when CantCompile - # Rewind stack_size using ctx.with_stack_size to allow stack_size changes - # before you return CantCompile. - @exit_compiler.compile_side_exit(jit.pc, ctx.with_stack_size(jit.stack_size_for_pc), asm) - - # If this is the first instruction, this block never needs to be invalidated. - if block.pc == iseq.body.iseq_encoded.to_i + index * C.VALUE.size - block.invalidated = true - end - - break - else - raise "compiling #{insn.name} returned unexpected status: #{status.inspect}" - end - end - - incr_counter(:compiled_block_count) - add_block(iseq, block) - end - - def leave_exit - @leave_exit ||= Assembler.new.then do |asm| - @exit_compiler.compile_leave_exit(asm) - @ocb.write(asm) - end - end - - def incr_counter(name) - if C.rjit_opts.stats - C.rb_rjit_counters[name][0] += 1 - end - end - - # Produce a generic context when the block version limit is hit for the block - def limit_block_versions(iseq, pc, ctx) - # Guard chains implement limits separately, do nothing - if ctx.chain_depth > 0 - return ctx.dup - end - - # If this block version we're about to add will hit the version limit - if list_blocks(iseq, pc).size + 1 >= MAX_VERSIONS - # Produce a generic context that stores no type information, - # but still respects the stack_size and sp_offset constraints. - # This new context will then match all future requests. - generic_ctx = Context.new - generic_ctx.stack_size = ctx.stack_size - generic_ctx.sp_offset = ctx.sp_offset - - if ctx.diff(generic_ctx) == TypeDiff::Incompatible - raise 'should substitute a compatible context' - end - - return generic_ctx - end - - return ctx.dup - end - - def list_blocks(iseq, pc) - rjit_blocks(iseq)[pc] - end - - # @param [Integer] pc - # @param [RubyVM::RJIT::Context] ctx - # @return [RubyVM::RJIT::Block,NilClass] - def find_block(iseq, pc, ctx) - versions = rjit_blocks(iseq)[pc] - - best_version = nil - best_diff = Float::INFINITY - - versions.each do |block| - # Note that we always prefer the first matching - # version found because of inline-cache chains - case ctx.diff(block.ctx) - in TypeDiff::Compatible[diff] if diff < best_diff - best_version = block - best_diff = diff - else - end - end - - return best_version - end - - # @param [RubyVM::RJIT::Block] block - def add_block(iseq, block) - rjit_blocks(iseq)[block.pc] << block - end - - # @param [RubyVM::RJIT::Block] block - def remove_block(iseq, block) - rjit_blocks(iseq)[block.pc].delete(block) - end - - def rjit_blocks(iseq) - # Guard against ISEQ GC at random moments - - unless C.imemo_type_p(iseq, C.imemo_iseq) - return Hash.new { |h, k| h[k] = [] } - end - - unless iseq.body.rjit_blocks - iseq.body.rjit_blocks = Hash.new { |blocks, pc| blocks[pc] = [] } - # For some reason, rb_rjit_iseq_mark didn't protect this Hash - # from being freed. So we rely on GC_REFS to keep the Hash. - GC_REFS << iseq.body.rjit_blocks - end - iseq.body.rjit_blocks - end - - def iseq_lineno(iseq, pc) - C.rb_iseq_line_no(iseq, (pc - iseq.body.iseq_encoded.to_i) / C.VALUE.size) - rescue RangeError # bignum too big to convert into `unsigned long long' (RangeError) - -1 - end - - # Verify the ctx's types and mappings against the compile-time stack, self, and locals. - # @param jit [RubyVM::RJIT::JITState] - # @param ctx [RubyVM::RJIT::Context] - def verify_ctx(jit, ctx) - # Only able to check types when at current insn - assert(jit.at_current_insn?) - - self_val = jit.peek_at_self - self_val_type = Type.from(self_val) - - # Verify self operand type - assert_compatible(self_val_type, ctx.get_opnd_type(SelfOpnd)) - - # Verify stack operand types - [ctx.stack_size, MAX_TEMP_TYPES].min.times do |i| - learned_mapping, learned_type = ctx.get_opnd_mapping(StackOpnd[i]) - stack_val = jit.peek_at_stack(i) - val_type = Type.from(stack_val) - - case learned_mapping - in MapToSelf - if C.to_value(self_val) != C.to_value(stack_val) - raise "verify_ctx: stack value was mapped to self, but values did not match:\n"\ - "stack: #{stack_val.inspect}, self: #{self_val.inspect}" - end - in MapToLocal[local_idx] - local_val = jit.peek_at_local(local_idx) - if C.to_value(local_val) != C.to_value(stack_val) - raise "verify_ctx: stack value was mapped to local, but values did not match:\n"\ - "stack: #{stack_val.inspect}, local: #{local_val.inspect}" - end - in MapToStack - # noop - end - - # If the actual type differs from the learned type - assert_compatible(val_type, learned_type) - end - - # Verify local variable types - local_table_size = jit.iseq.body.local_table_size - [local_table_size, MAX_TEMP_TYPES].min.times do |i| - learned_type = ctx.get_local_type(i) - local_val = jit.peek_at_local(i) - local_type = Type.from(local_val) - - assert_compatible(local_type, learned_type) - end - end - - def assert_compatible(actual_type, ctx_type) - if actual_type.diff(ctx_type) == TypeDiff::Incompatible - raise "verify_ctx: ctx type (#{ctx_type.type.inspect}) is incompatible with actual type (#{actual_type.type.inspect})" - end - end - - def assert(cond) - unless cond - raise "'#{cond.inspect}' was not true" - end - end - - def supported_platform? - return @supported_platform if defined?(@supported_platform) - @supported_platform = RUBY_PLATFORM.match?(/x86_64/).tap do |supported| - warn "warning: RJIT does not support #{RUBY_PLATFORM} yet" unless supported - end - end - end -end |