diff options
-rw-r--r-- | yjit/src/codegen.rs | 225 | ||||
-rw-r--r-- | yjit/src/core.rs | 1228 | ||||
-rw-r--r-- | yjit/src/disasm.rs | 26 | ||||
-rw-r--r-- | yjit/src/invariants.rs | 119 | ||||
-rw-r--r-- | yjit/src/stats.rs | 4 |
5 files changed, 965 insertions, 637 deletions
diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 31582c26bc..e9c17cb537 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -12,6 +12,7 @@ use crate::utils::*; use CodegenStatus::*; use YARVOpnd::*; +use std::cell::Cell; use std::cmp; use std::collections::HashMap; use std::ffi::CStr; @@ -38,63 +39,89 @@ type InsnGenFn = fn( ocb: &mut OutlinedCb, ) -> CodegenStatus; -/// Code generation state -/// This struct only lives while code is being generated +/// Ephemeral code generation state. +/// Represents a [core::Block] while we build it. pub struct JITState { - // Block version being compiled - block: BlockRef, - - // Instruction sequence this is associated with + /// Instruction sequence for the compiling block iseq: IseqPtr, - // Index of the current instruction being compiled - insn_idx: u16, + /// The iseq index of the first instruction in the block + starting_insn_idx: IseqIdx, + + /// The [Context] entering into the first instruction of the block + starting_ctx: Context, + + /// The placement for the machine code of the [Block] + output_ptr: CodePtr, - // Opcode for the instruction being compiled + /// Index of the current instruction being compiled + insn_idx: IseqIdx, + + /// Opcode for the instruction being compiled opcode: usize, - // PC of the instruction being compiled + /// PC of the instruction being compiled pc: *mut VALUE, - // Side exit to the instruction being compiled. See :side-exit:. + /// Side exit to the instruction being compiled. See :side-exit:. side_exit_for_pc: Option<CodePtr>, - // Execution context when compilation started - // This allows us to peek at run-time values + /// Execution context when compilation started + /// This allows us to peek at run-time values ec: EcPtr, - // Whether we need to record the code address at - // the end of this bytecode instruction for global invalidation - record_boundary_patch_point: bool, + /// The outgoing branches the block will have + pub pending_outgoing: Vec<PendingBranchRef>, + + // --- Fields for block invalidation and invariants tracking below: + // Public mostly so into_block defined in the sibling module core + // can partially move out of Self. - // The block's outgoing branches - outgoing: Vec<BranchRef>, + /// Whether we need to record the code address at + /// the end of this bytecode instruction for global invalidation + pub record_boundary_patch_point: bool, - // The block's CME dependencies - cme_dependencies: Vec<CmePtr>, + /// Code for immediately exiting upon entry to the block. + /// Required for invalidation. + pub block_entry_exit: Option<CodePtr>, + + /// A list of callable method entries that must be valid for the block to be valid. + pub method_lookup_assumptions: Vec<CmePtr>, + + /// A list of basic operators that not be redefined for the block to be valid. + pub bop_assumptions: Vec<(RedefinitionFlag, ruby_basic_operators)>, + + /// A list of constant expression path segments that must have + /// not been written to for the block to be valid. + pub stable_constant_names_assumption: Option<*const ID>, + + /// When true, the block is valid only when there is a total of one ractor running + pub block_assumes_single_ractor: bool, } impl JITState { - pub fn new(blockref: &BlockRef, ec: EcPtr) -> Self { + pub fn new(blockid: BlockId, starting_ctx: Context, output_ptr: CodePtr, ec: EcPtr) -> Self { JITState { - block: blockref.clone(), - iseq: ptr::null(), // TODO: initialize this from the blockid + iseq: blockid.iseq, + starting_insn_idx: blockid.idx, + starting_ctx, + output_ptr, insn_idx: 0, opcode: 0, pc: ptr::null_mut::<VALUE>(), side_exit_for_pc: None, + pending_outgoing: vec![], ec, record_boundary_patch_point: false, - outgoing: Vec::new(), - cme_dependencies: Vec::new(), + block_entry_exit: None, + method_lookup_assumptions: vec![], + bop_assumptions: vec![], + stable_constant_names_assumption: None, + block_assumes_single_ractor: false, } } - pub fn get_block(&self) -> BlockRef { - self.block.clone() - } - - pub fn get_insn_idx(&self) -> u16 { + pub fn get_insn_idx(&self) -> IseqIdx { self.insn_idx } @@ -110,6 +137,18 @@ impl JITState { self.pc } + pub fn get_starting_insn_idx(&self) -> IseqIdx { + self.starting_insn_idx + } + + pub fn get_block_entry_exit(&self) -> Option<CodePtr> { + self.block_entry_exit + } + + pub fn get_starting_ctx(&self) -> Context { + self.starting_ctx.clone() + } + pub fn get_arg(&self, arg_idx: isize) -> VALUE { // insn_len require non-test config #[cfg(not(test))] @@ -175,18 +214,22 @@ impl JITState { } } + pub fn assume_method_lookup_stable(&mut self, ocb: &mut OutlinedCb, cme: CmePtr) { + jit_ensure_block_entry_exit(self, ocb); + self.method_lookup_assumptions.push(cme); + } + fn get_cfp(&self) -> *mut rb_control_frame_struct { unsafe { get_ec_cfp(self.ec) } } - // Push an outgoing branch ref - pub fn push_outgoing(&mut self, branch: BranchRef) { - self.outgoing.push(branch); + pub fn assume_stable_constant_names(&mut self, ocb: &mut OutlinedCb, id: *const ID) { + jit_ensure_block_entry_exit(self, ocb); + self.stable_constant_names_assumption = Some(id); } - // Push a CME dependency - pub fn push_cme_dependency(&mut self, cme: CmePtr) { - self.cme_dependencies.push(cme); + pub fn queue_outgoing_branch(&mut self, branch: PendingBranchRef) { + self.pending_outgoing.push(branch) } } @@ -498,22 +541,20 @@ fn get_side_exit(jit: &mut JITState, ocb: &mut OutlinedCb, ctx: &Context) -> Tar // Ensure that there is an exit for the start of the block being compiled. // Block invalidation uses this exit. pub fn jit_ensure_block_entry_exit(jit: &mut JITState, ocb: &mut OutlinedCb) { - let blockref = jit.block.clone(); - let mut block = blockref.borrow_mut(); - let block_ctx = block.get_ctx(); - let blockid = block.get_blockid(); - - if block.entry_exit.is_some() { + if jit.block_entry_exit.is_some() { return; } + let block_starting_context = &jit.get_starting_ctx(); + // If we're compiling the first instruction in the block. - if jit.insn_idx == blockid.idx { + if jit.insn_idx == jit.starting_insn_idx { // Generate the exit with the cache in jitstate. - block.entry_exit = Some(get_side_exit(jit, ocb, &block_ctx).unwrap_code_ptr()); + let entry_exit = get_side_exit(jit, ocb, block_starting_context).unwrap_code_ptr(); + jit.block_entry_exit = Some(entry_exit); } else { - let block_entry_pc = unsafe { rb_iseq_pc_at_idx(blockid.iseq, blockid.idx.into()) }; - block.entry_exit = Some(gen_outlined_exit(block_entry_pc, &block_ctx, ocb)); + let block_entry_pc = unsafe { rb_iseq_pc_at_idx(jit.iseq, jit.starting_insn_idx.into()) }; + jit.block_entry_exit = Some(gen_outlined_exit(block_entry_pc, block_starting_context, ocb)); } } @@ -725,18 +766,18 @@ pub fn gen_single_block( verify_blockid(blockid); assert!(!(blockid.idx == 0 && ctx.get_stack_size() > 0)); + // Save machine code placement of the block. `cb` might page switch when we + // generate code in `ocb`. + let block_start_addr = cb.get_write_ptr(); + // Instruction sequence to compile let iseq = blockid.iseq; let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; let iseq_size: u16 = iseq_size.try_into().unwrap(); let mut insn_idx: u16 = blockid.idx; - let starting_insn_idx = insn_idx; - - // Allocate the new block - let blockref = Block::new(blockid, &ctx, cb.get_write_ptr()); // Initialize a JIT state object - let mut jit = JITState::new(&blockref, ec); + let mut jit = JITState::new(blockid, ctx.clone(), cb.get_write_ptr(), ec); jit.iseq = blockid.iseq; // Create a backend assembler instance @@ -762,7 +803,7 @@ pub fn gen_single_block( // We need opt_getconstant_path to be in a block all on its own. Cut the block short // if we run into it. This is necessary because we want to invalidate based on the // instruction's index. - if opcode == YARVINSN_opt_getconstant_path.as_usize() && insn_idx > starting_insn_idx { + if opcode == YARVINSN_opt_getconstant_path.as_usize() && insn_idx > jit.starting_insn_idx { jump_to_next_insn(&mut jit, &ctx, &mut asm, ocb); break; } @@ -814,17 +855,15 @@ pub fn gen_single_block( println!("can't compile {}", insn_name(opcode)); } - let mut block = jit.block.borrow_mut(); - // TODO: if the codegen function makes changes to ctx and then return YJIT_CANT_COMPILE, // the exit this generates would be wrong. We could save a copy of the entry context // and assert that ctx is the same here. gen_exit(jit.pc, &ctx, &mut asm); - // If this is the first instruction in the block, then we can use - // the exit for block->entry_exit. - if insn_idx == block.get_blockid().idx { - block.entry_exit = Some(block.get_start_addr()); + // If this is the first instruction in the block, then + // the entry address is the address for block_entry_exit + if insn_idx == jit.starting_insn_idx { + jit.block_entry_exit = Some(jit.output_ptr); } break; @@ -842,45 +881,28 @@ pub fn gen_single_block( break; } } + let end_insn_idx = insn_idx; - // Finish filling out the block - { - let mut block = jit.block.borrow_mut(); - if block.entry_exit.is_some() { - asm.pad_inval_patch(); - } - - // Compile code into the code block - let gc_offsets = asm.compile(cb); - - // Add the GC offsets to the block - block.set_gc_obj_offsets(gc_offsets); - - // Set CME dependencies to the block - block.set_cme_dependencies(jit.cme_dependencies); - - // Set outgoing branches to the block - block.set_outgoing(jit.outgoing); - - // Mark the end position of the block - block.set_end_addr(cb.get_write_ptr()); + // We currently can't handle cases where the request is for a block that + // doesn't go to the next instruction in the same iseq. + assert!(!jit.record_boundary_patch_point); - // Store the index of the last instruction in the block - block.set_end_idx(insn_idx); + // Pad the block if it has the potential to be invalidated + if jit.block_entry_exit.is_some() { + asm.pad_inval_patch(); } - // We currently can't handle cases where the request is for a block that - // doesn't go to the next instruction. - assert!(!jit.record_boundary_patch_point); + // Compile code into the code block + let gc_offsets = asm.compile(cb); + let end_addr = cb.get_write_ptr(); // If code for the block doesn't fit, fail if cb.has_dropped_bytes() || ocb.unwrap().has_dropped_bytes() { - free_block(&blockref); return Err(()); } // Block compiled successfully - Ok(blockref) + Ok(jit.into_block(end_insn_idx, block_start_addr, end_addr, gc_offsets)) } fn gen_nop( @@ -3595,7 +3617,7 @@ fn gen_branchif( ctx, Some(next_block), Some(ctx), - BranchGenFn::BranchIf(BranchShape::Default), + BranchGenFn::BranchIf(Cell::new(BranchShape::Default)), ); } @@ -3652,7 +3674,7 @@ fn gen_branchunless( ctx, Some(next_block), Some(ctx), - BranchGenFn::BranchUnless(BranchShape::Default), + BranchGenFn::BranchUnless(Cell::new(BranchShape::Default)), ); } @@ -3706,7 +3728,7 @@ fn gen_branchnil( ctx, Some(next_block), Some(ctx), - BranchGenFn::BranchNil(BranchShape::Default), + BranchGenFn::BranchNil(Cell::new(BranchShape::Default)), ); } @@ -4533,7 +4555,7 @@ fn jit_obj_respond_to( // Invalidate this block if method lookup changes for the method being queried. This works // both for the case where a method does or does not exist, as for the latter we asked for a // "negative CME" earlier. - assume_method_lookup_stable(jit, ocb, target_cme); + jit.assume_method_lookup_stable(ocb, target_cme); // Generate a side exit let side_exit = get_side_exit(jit, ocb, ctx); @@ -6308,7 +6330,7 @@ fn gen_send_general( // Register block for invalidation //assert!(cme->called_id == mid); - assume_method_lookup_stable(jit, ocb, cme); + jit.assume_method_lookup_stable(ocb, cme); // To handle the aliased method case (VM_METHOD_TYPE_ALIAS) loop { @@ -6478,7 +6500,7 @@ fn gen_send_general( flags |= VM_CALL_FCALL | VM_CALL_OPT_SEND; - assume_method_lookup_stable(jit, ocb, cme); + jit.assume_method_lookup_stable(ocb, cme); let (known_class, type_mismatch_exit) = { if compile_time_name.string_p() { @@ -6973,8 +6995,8 @@ fn gen_invokesuper( // We need to assume that both our current method entry and the super // method entry we invoke remain stable - assume_method_lookup_stable(jit, ocb, me); - assume_method_lookup_stable(jit, ocb, cme); + jit.assume_method_lookup_stable(ocb, me); + jit.assume_method_lookup_stable(ocb, cme); // Method calls may corrupt types ctx.clear_local_types(); @@ -7428,14 +7450,13 @@ fn gen_opt_getconstant_path( asm.store(stack_top, ic_entry_val); } else { // Optimize for single ractor mode. - // FIXME: This leaks when st_insert raises NoMemoryError if !assume_single_ractor_mode(jit, ocb) { return CantCompile; } // Invalidate output code on any constant writes associated with // constants referenced within the current block. - assume_stable_constant_names(jit, ocb, idlist); + jit.assume_stable_constant_names(ocb, idlist); jit_putobject(jit, ctx, asm, unsafe { (*ice).value }); } @@ -8112,15 +8133,15 @@ mod tests { use super::*; fn setup_codegen() -> (JITState, Context, Assembler, CodeBlock, OutlinedCb) { - let blockid = BlockId { - iseq: ptr::null(), - idx: 0, - }; let cb = CodeBlock::new_dummy(256 * 1024); - let block = Block::new(blockid, &Context::default(), cb.get_write_ptr()); return ( - JITState::new(&block, ptr::null()), // No execution context in tests. No peeking! + JITState::new( + BlockId { iseq: std::ptr::null(), idx: 0 }, + Context::default(), + cb.get_write_ptr(), + ptr::null(), // No execution context in tests. No peeking! + ), Context::default(), Assembler::new(), cb, diff --git a/yjit/src/core.rs b/yjit/src/core.rs index 5c8bf32e86..888f795279 100644 --- a/yjit/src/core.rs +++ b/yjit/src/core.rs @@ -1,3 +1,4 @@ +//! Code versioning, retained live control flow graph mutations, type tracking, etc. use crate::asm::*; use crate::backend::ir::*; use crate::codegen::*; @@ -11,12 +12,16 @@ use crate::disasm::*; use core::ffi::c_void; use std::cell::*; use std::collections::HashSet; -use std::hash::{Hash, Hasher}; +use std::fmt; use std::mem; -use std::rc::{Rc}; +use std::ops::Range; +use std::rc::Rc; +use mem::MaybeUninit; +use std::ptr; +use ptr::NonNull; use YARVOpnd::*; use TempMapping::*; -use crate::invariants::block_assumptions_free; +use crate::invariants::*; // Maximum number of temp value types we keep track of pub const MAX_TEMP_TYPES: usize = 8; @@ -24,6 +29,10 @@ pub const MAX_TEMP_TYPES: usize = 8; // Maximum number of local variable types we keep track of const MAX_LOCAL_TYPES: usize = 8; +/// An index into `ISEQ_BODY(iseq)->iseq_encoded`. Points +/// to a YARV instruction or an instruction operand. +pub type IseqIdx = u16; + // Represent the type of a value (local/stack/self) in YJIT #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum Type { @@ -410,12 +419,12 @@ pub enum BranchShape { Default, // Neither target is next } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub enum BranchGenFn { - BranchIf(BranchShape), - BranchNil(BranchShape), - BranchUnless(BranchShape), - JumpToTarget0(BranchShape), + BranchIf(Cell<BranchShape>), + BranchNil(Cell<BranchShape>), + BranchUnless(Cell<BranchShape>), + JumpToTarget0(Cell<BranchShape>), JNZToTarget0, JZToTarget0, JBEToTarget0, @@ -423,10 +432,10 @@ pub enum BranchGenFn { } impl BranchGenFn { - pub fn call(self, asm: &mut Assembler, target0: CodePtr, target1: Option<CodePtr>) { + pub fn call(&self, asm: &mut Assembler, target0: CodePtr, target1: Option<CodePtr>) { match self { BranchGenFn::BranchIf(shape) => { - match shape { + match shape.get() { BranchShape::Next0 => asm.jz(target1.unwrap().into()), BranchShape::Next1 => asm.jnz(target0.into()), BranchShape::Default => { @@ -436,7 +445,7 @@ impl BranchGenFn { } } BranchGenFn::BranchNil(shape) => { - match shape { + match shape.get() { BranchShape::Next0 => asm.jne(target1.unwrap().into()), BranchShape::Next1 => asm.je(target0.into()), BranchShape::Default => { @@ -446,7 +455,7 @@ impl BranchGenFn { } } BranchGenFn::BranchUnless(shape) => { - match shape { + match shape.get() { BranchShape::Next0 => asm.jnz(target1.unwrap().into()), BranchShape::Next1 => asm.jz(target0.into()), BranchShape::Default => { @@ -456,10 +465,10 @@ impl BranchGenFn { } } BranchGenFn::JumpToTarget0(shape) => { - if shape == BranchShape::Next1 { + if shape.get() == BranchShape::Next1 { panic!("Branch shape Next1 not allowed in JumpToTarget0!"); } - if shape == BranchShape::Default { + if shape.get() == BranchShape::Default { asm.jmp(target0.into()); } } @@ -479,12 +488,12 @@ impl BranchGenFn { } } - pub fn get_shape(self) -> BranchShape { + pub fn get_shape(&self) -> BranchShape { match self { BranchGenFn::BranchIf(shape) | BranchGenFn::BranchNil(shape) | BranchGenFn::BranchUnless(shape) | - BranchGenFn::JumpToTarget0(shape) => shape, + BranchGenFn::JumpToTarget0(shape) => shape.get(), BranchGenFn::JNZToTarget0 | BranchGenFn::JZToTarget0 | BranchGenFn::JBEToTarget0 | @@ -492,18 +501,18 @@ impl BranchGenFn { } } - pub fn set_shape(&mut self, new_shape: BranchShape) { + pub fn set_shape(&self, new_shape: BranchShape) { match self { BranchGenFn::BranchIf(shape) | BranchGenFn::BranchNil(shape) | BranchGenFn::BranchUnless(shape) => { - *shape = new_shape; + shape.set(new_shape); } BranchGenFn::JumpToTarget0(shape) => { if new_shape == BranchShape::Next1 { panic!("Branch shape Next1 not allowed in JumpToTarget0!"); } - *shape = new_shape; + shape.set(new_shape); } BranchGenFn::JNZToTarget0 | BranchGenFn::JZToTarget0 | @@ -516,7 +525,7 @@ impl BranchGenFn { } /// A place that a branch could jump to -#[derive(Debug)] +#[derive(Debug, Clone)] enum BranchTarget { Stub(Box<BranchStub>), // Not compiled yet Block(BlockRef), // Already compiled @@ -526,79 +535,108 @@ impl BranchTarget { fn get_address(&self) -> Option<CodePtr> { match self { BranchTarget::Stub(stub) => stub.address, - BranchTarget::Block(blockref) => Some(blockref.borrow().start_addr), + BranchTarget::Block(blockref) => Some(unsafe { blockref.as_ref() }.start_addr), } } fn get_blockid(&self) -> BlockId { match self { - BranchTarget::Stub(stub) => stub.id, - BranchTarget::Block(blockref) => blockref.borrow().blockid, + BranchTarget::Stub(stub) => BlockId { iseq: stub.iseq.get(), idx: stub.iseq_idx }, + BranchTarget::Block(blockref) => unsafe { blockref.as_ref() }.get_blockid(), } } fn get_ctx(&self) -> Context { match self { BranchTarget::Stub(stub) => stub.ctx.clone(), - BranchTarget::Block(blockref) => blockref.borrow().ctx.clone(), + BranchTarget::Block(blockref) => unsafe { blockref.as_ref() }.ctx.clone(), } } fn get_block(&self) -> Option<BlockRef> { match self { BranchTarget::Stub(_) => None, - BranchTarget::Block(blockref) => Some(blockref.clone()), + BranchTarget::Block(blockref) => Some(*blockref), } } - fn set_iseq(&mut self, iseq: IseqPtr) { + fn set_iseq(&self, iseq: IseqPtr) { match self { - BranchTarget::Stub(stub) => stub.id.iseq = iseq, - BranchTarget::Block(blockref) => blockref.borrow_mut().blockid.iseq = iseq, + BranchTarget::Stub(stub) => stub.iseq.set(iseq), + BranchTarget::Block(blockref) => unsafe { blockref.as_ref() }.iseq.set(iseq), } } } -#[derive(Debug)] +#[derive(Debug, Clone)] struct BranchStub { address: Option<CodePtr>, - id: BlockId, + iseq: Cell<IseqPtr>, + iseq_idx: IseqIdx, ctx: Context, } /// Store info about an outgoing branch in a code segment /// Note: care must be taken to minimize the size of branch objects -#[derive(Debug)] pub struct Branch { // Block this is attached to block: BlockRef, // Positions where the generated code starts and ends - start_addr: Option<CodePtr>, - end_addr: Option<CodePtr>, // exclusive + start_addr: CodePtr, + end_addr: Cell<CodePtr>, // exclusive // Branch target blocks and their contexts - targets: [Option<Box<BranchTarget>>; 2], + targets: [Cell<Option<Box<BranchTarget>>>; 2], // Branch code generation function gen_fn: BranchGenFn, } +/// A [Branch] for a [Block] that is under construction. +/// Fields correspond, but may be `None` during construction. +pub struct PendingBranch { + /// Allocation holder for the address of the constructed branch + /// in error paths Box deallocates it. + uninit_branch: Box<MaybeUninit<Branch>>, + + /// Branch code generation function + gen_fn: BranchGenFn, + + /// Positions where the generated code starts and ends + start_addr: Cell<Option<CodePtr>>, + end_addr: Cell<Option<CodePtr>>, // exclusive + + /// Branch target blocks and their contexts + targets: [Cell<Option<Box<BranchTarget>>>; 2], +} + impl Branch { // Compute the size of the branch code fn code_size(&self) -> usize { - (self.end_addr.unwrap().raw_ptr() as usize) - (self.start_addr.unwrap().raw_ptr() as usize) + (self.end_addr.get().raw_ptr() as usize) - (self.start_addr.raw_ptr() as usize) } /// Get the address of one of the branch destination fn get_target_address(&self, target_idx: usize) -> Option<CodePtr> { - self.targets[target_idx].as_ref().and_then(|target| target.get_address()) + unsafe { + self.targets[target_idx] + .ref_unchecked() + .as_ref() + .and_then(|target| target.get_address()) + } } fn get_stub_count(&self) -> usize { let mut count = 0; - for target in self.targets.iter().flatten() { - if let BranchTarget::Stub(_) = target.as_ref() { + for target in self.targets.iter() { + if unsafe { + // SAFETY: no mutation + matches!( + target.ref_unchecked().as_ref().map(Box::as_ref), + Some(BranchTarget::Stub(_)) + ) + } { count += 1; } } @@ -606,19 +644,117 @@ impl Branch { } } +impl std::fmt::Debug for Branch { + // Can't derive this because `targets: !Copy` due to Cell. + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let targets = unsafe { + // SAFETY: + // While the references are live for the result of this function, + // no mutation happens because we are only calling derived fmt::Debug functions. + [self.targets[0].as_ptr().as_ref().unwrap(), self.targets[1].as_ptr().as_ref().unwrap()] + }; + + formatter + .debug_struct("Branch") + .field("block", &self.block) + .field("start", &self.start_addr) + .field("end", &self.end_addr) + .field("targets", &targets) + .field("gen_fn", &self.gen_fn) + .finish() + } +} + +impl PendingBranch { + /// Set up a branch target at `target_idx`. Find an existing block to branch to + /// or generate a stub for one. + fn set_target( + &self, + target_idx: u32, + target: BlockId, + ctx: &Context, + ocb: &mut OutlinedCb, + ) -> Option<CodePtr> { + // If the block already exists + if let Some(blockref) = find_block_version(target, ctx) { + let block = unsafe { blockref.as_ref() }; + + // Fill out the target with this block + self.targets[target_idx.as_usize()] + .set(Some(Box::new(BranchTarget::Block(blockref)))); + return Some(block.start_addr); + } + + // The branch struct is uninitialized right now but as a stable address. + // We make sure the stub runs after the branch is initialized. + let branch_struct_addr = self.uninit_branch.as_ptr() as usize; + let stub_addr = gen_call_branch_stub_hit(ocb, branch_struct_addr, target_idx); + + if let Some(stub_addr) = stub_addr { + // Fill the branch target with a stub + self.targets[target_idx.as_usize()].set(Some(Box::new(BranchTarget::Stub(Box::new(BranchStub { + address: Some(stub_addr), + iseq: Cell::new(target.iseq), + iseq_idx: target.idx, + ctx: ctx.clone(), + }))))); + } + + stub_addr + } + + // Construct the branch and wire it up in the grpah + fn into_branch(mut self, uninit_block: BlockRef) -> BranchRef { + // Make the branch + let branch = Branch { + block: uninit_block, + start_addr: self.start_addr.get().unwrap(), + end_addr: Cell::new(self.end_addr.get().unwrap()), + targets: self.targets, + gen_fn: self.gen_fn, + }; + // Move it to the designated place on + // the heap and unwrap MaybeUninit. + self.uninit_branch.write(branch); + let raw_branch: *mut MaybeUninit<Branch> = Box::into_raw(self.uninit_branch); + let branchref = NonNull::new(raw_branch as *mut Branch).expect("no null from Box"); + + // SAFETY: just allocated it + let branch = unsafe { branchref.as_ref() }; + // For block branch targets, put the new branch in the + // appropriate incoming list. + for target in branch.targets.iter() { + // SAFETY: no mutation + let out_block: Option<BlockRef> = unsafe { + target.ref_unchecked().as_ref().and_then(|target| target.get_block()) + }; + + if let Some(out_block) = out_block { + // SAFETY: These blockrefs come from set_target() which only puts blocks from + // ISeqs, which are all initialized. Note that uninit_block isn't in any ISeq + // payload yet. + unsafe { out_block.as_ref() }.incoming.push(branchref); + } + } + + branchref + } +} + // In case a block is invalidated, this helps to remove all pointers to the block. pub type CmePtr = *const rb_callable_method_entry_t; /// Basic block version /// Represents a portion of an iseq compiled with a given context /// Note: care must be taken to minimize the size of block_t objects -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct Block { - // Bytecode sequence (iseq, idx) this is a version of - blockid: BlockId, + // The byte code instruction sequence this is a version of. + // Can change due to moving GC. + iseq: Cell<IseqPtr>, - // Index one past the last instruction for this block in the iseq - end_idx: u16, + // Index range covered by this version in `ISEQ_BODY(iseq)->iseq_encoded`. + iseq_range: Range<IseqIdx>, // Context at the start of the block // This should never be mutated @@ -626,11 +762,11 @@ pub struct Block { // Positions where the generated code starts and ends start_addr: CodePtr, - end_addr: Option<CodePtr>, + end_addr: Cell<CodePtr>, // List of incoming branches (from predecessors) // These are reference counted (ownership shared between predecessor and successors) - incoming: Vec<BranchRef>, + incoming: MutableBranchList, // NOTE: we might actually be able to store the branches here without refcounting // however, using a RefCell makes it easy to get a pointer to Branch objects @@ -644,20 +780,38 @@ pub struct Block { // CME dependencies of this block, to help to remove all pointers to this // block in the system. - cme_dependencies: Box<[CmePtr]>, + cme_dependencies: Box<[Cell<CmePtr>]>, // Code address of an exit for `ctx` and `blockid`. // Used for block invalidation. - pub entry_exit: Option<CodePtr>, -} - -/// Reference-counted pointer to a block that can be borrowed mutably. -/// Wrapped so we could implement [Hash] and [Eq] for use with stdlib collections. -#[derive(Debug)] -pub struct BlockRef(Rc<RefCell<Block>>); - -/// Reference-counted pointer to a branch that can be borrowed mutably -pub type BranchRef = Rc<RefCell<Branch>>; + entry_exit: Option<CodePtr>, +} + +/// Pointer to a [Block]. +/// +/// # Safety +/// +/// _Never_ derive a `&mut Block` from this and always use +/// [std::ptr::NonNull::as_ref] to get a `&Block`. `&'a mut` +/// in Rust asserts that there are no other references live +/// over the lifetime `'a`. This uniqueness assertion does +/// not hold in many situations for us, even when you ignore +/// the fact that our control flow graph can have cycles. +/// Here are just two examples where we have overlapping references: +/// - Yielding to a different OS thread within the same +/// ractor during compilation +/// - The GC calling [rb_yjit_iseq_mark] during compilation +/// +/// Technically, for soundness, we also need to ensure that +/// the we have the VM lock while the result of `as_ref()` +/// is live, so that no deallocation happens while the +/// shared reference is live. The vast majority of our code run while +/// holding the VM lock, though. +pub type BlockRef = NonNull<Block>; + +/// Pointer to a [Branch]. See [BlockRef] for notes about +/// proper usage. +pub type BranchRef = NonNull<Branch>; /// List of block versions for a given blockid type VersionList = Vec<BlockRef>; @@ -666,47 +820,33 @@ type VersionList = Vec<BlockRef>; /// An instance of this is stored on each iseq type VersionMap = Vec<VersionList>; -impl BlockRef { - /// Constructor - pub fn new(rc: Rc<RefCell<Block>>) -> Self { - Self(rc) - } - - /// Borrow the block through [RefCell]. - pub fn borrow(&self) -> Ref<'_, Block> { - self.0.borrow() - } - - /// Borrow the block for mutation through [RefCell]. - pub fn borrow_mut(&self) -> RefMut<'_, Block> { - self.0.borrow_mut() - } -} +/// [Interior mutability][1] wrapper for a list of branches. +/// O(n) insertion, but space efficient. We generally expect +/// blocks to have only a few branches. +/// +/// [1]: https://doc.rust-lang.org/std/cell/struct.UnsafeCell.html +#[repr(transparent)] +struct MutableBranchList(Cell<Box<[BranchRef]>>); -impl Clone for BlockRef { - /// Clone the [Rc] - fn clone(&self) -> Self { - Self(self.0.clone()) +impl MutableBranchList { + fn push(&self, branch: BranchRef) { + // Temporary move the boxed slice out of self. + // oom=abort is load bearing here... + let mut current_list = self.0.take().into_vec(); + current_list.push(branch); + self.0.set(current_list.into_boxed_slice()); } } -impl Hash for BlockRef { - /// Hash the reference by hashing the pointer - fn hash<H: Hasher>(&self, state: &mut H) { - let rc_ptr = Rc::as_ptr(&self.0); - rc_ptr.hash(state); - } -} +impl fmt::Debug for MutableBranchList { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + // SAFETY: the derived Clone for boxed slices does not mutate this Cell + let branches = unsafe { self.0.ref_unchecked().clone() }; -impl PartialEq for BlockRef { - /// Equality defined by allocation identity - fn eq(&self, other: &Self) -> bool { - Rc::ptr_eq(&self.0, &other.0) + formatter.debug_list().entries(branches.into_iter()).finish() } } -/// It's comparison by identity so all the requirements are satisfied -impl Eq for BlockRef {} /// This is all the data YJIT stores on an iseq /// This will be dynamically allocated by C code @@ -845,15 +985,21 @@ pub extern "C" fn rb_yjit_iseq_free(payload: *mut c_void) { // SAFETY: We got the pointer from Box::into_raw(). let payload = unsafe { Box::from_raw(payload) }; - // Increment the freed iseq count - incr_counter!(freed_iseq_count); - - // Free all blocks in the payload + // Free all blocks in version_map. The GC doesn't free running iseqs. for versions in &payload.version_map { for block in versions { - free_block(block); + // SAFETY: blocks in the version_map are always well connected + unsafe { free_block(*block, true) }; } } + + // Free dead blocks + for block in payload.dead_blocks { + unsafe { free_block(block, false) }; + } + + // Increment the freed iseq count + incr_counter!(freed_iseq_count); } /// GC callback for marking GC objects in the the per-iseq payload. @@ -880,20 +1026,26 @@ pub extern "C" fn rb_yjit_iseq_mark(payload: *mut c_void) { for versions in &payload.version_map { for block in versions { - let block = block.borrow(); + // SAFETY: all blocks inside version_map are initialized. + let block = unsafe { block.as_ref() }; - unsafe { rb_gc_mark_movable(block.blockid.iseq.into()) }; + unsafe { rb_gc_mark_movable(block.iseq.get().into()) }; // Mark method entry dependencies - for &cme_dep in block.cme_dependencies.iter() { - unsafe { rb_gc_mark_movable(cme_dep.into()) }; + for cme_dep in block.cme_dependencies.iter() { + unsafe { rb_gc_mark_movable(cme_dep.get().into()) }; } // Mark outgoing branch entries for branch in block.outgoing.iter() { - let branch = branch.borrow(); - for target in branch.targets.iter().flatten() { - unsafe { rb_gc_mark_movable(target.get_blockid().iseq.into()) }; + let branch = unsafe { branch.as_ref() }; + for target in branch.targets.iter() { + // SAFETY: no mutation inside unsafe + let target_iseq = unsafe { target.ref_unchecked().as_ref().map(|target| target.get_blockid().iseq) }; + + if let Some(target_iseq) = target_iseq { + unsafe { rb_gc_mark_movable(target_iseq.into()) }; + } } } @@ -941,13 +1093,16 @@ pub extern "C" fn rb_yjit_iseq_update_references(payload: *mut c_void) { for versions in &payload.version_map { for version in versions { - let mut block = version.borrow_mut(); + // SAFETY: all blocks inside version_map are initialized + let block = unsafe { version.as_ref() }; - block.blockid.iseq = unsafe { rb_gc_location(block.blockid.iseq.into()) }.as_iseq(); + block.iseq.set(unsafe { rb_gc_location(block.iseq.get().into()) }.as_iseq()); // Update method entry dependencies - for cme_dep in block.cme_dependencies.iter_mut() { - *cme_dep = unsafe { rb_gc_location((*cme_dep).into()) }.as_cme(); + for cme_dep in block.cme_dependencies.iter() { + let cur_cme: VALUE = cme_dep.get().into(); + let new_cme = unsafe { rb_gc_location(cur_cme) }.as_cme(); + cme_dep.set(new_cme); } // Walk over references to objects in generated code. @@ -973,12 +1128,19 @@ pub extern "C" fn rb_yjit_iseq_update_references(payload: *mut c_void) { } // Update outgoing branch entries - let outgoing_branches = block.outgoing.clone(); // clone to use after borrow - mem::drop(block); // end mut borrow: target.set_iseq and target.get_blockid() might (mut) borrow it - for branch in outgoing_branches.iter() { - let mut branch = branch.borrow_mut(); - for target in branch.targets.iter_mut().flatten() { - target.set_iseq(unsafe { rb_gc_location(target.get_blockid().iseq.into()) }.as_iseq()); + for branch in block.outgoing.iter() { + let branch = unsafe { branch.as_ref() }; + for target in branch.targets.iter() { + // SAFETY: no mutation inside unsafe + let current_iseq = unsafe { target.ref_unchecked().as_ref().map(|target| target.get_blockid().iseq) }; + + if let Some(current_iseq) = current_iseq { + let updated_iseq = unsafe { rb_gc_location(current_iseq.into()) } + .as_iseq(); + // SAFETY: the Cell::set is not on the reference given out + // by ref_unchecked. + unsafe { target.ref_unchecked().as_ref().unwrap().set_iseq(updated_iseq) }; + } } } } @@ -1058,7 +1220,7 @@ pub fn get_or_create_iseq_block_list(iseq: IseqPtr) -> Vec<BlockRef> { // For each version at this instruction index for version in version_list { // Clone the block ref and add it to the list - blocks.push(version.clone()); + blocks.push(*version); } } @@ -1078,13 +1240,14 @@ fn find_block_version(blockid: BlockId, ctx: &Context) -> Option<BlockRef> { let mut best_diff = usize::MAX; // For each version matching the blockid - for blockref in versions.iter_mut() { - let block = blockref.borrow(); + for blockref in versions.iter() { + let block = unsafe { blockref.as_ref() }; + // Note that we always prefer the first matching // version found because of inline-cache chains match ctx.diff(&block.ctx) { TypeDiff::Compatible(diff) if diff < best_diff => { - best_version = Some(blockref.clone()); + best_version = Some(*blockref); best_diff = diff; } _ => {} @@ -1130,23 +1293,40 @@ pub fn limit_block_versions(blockid: BlockId, ctx: &Context) -> Context { return ctx.clone(); } -/// Keep track of a block version. Block should be fully constructed. -/// Uses `cb` for running write barriers. -fn add_block_version(blockref: &BlockRef, cb: &CodeBlock) { - let block = blockref.borrow(); +/// Install a block version into its [IseqPayload], letting the GC track its +/// lifetime, and allowing it to be considered for use for other +/// blocks we might generate. Uses `cb` for running write barriers. +/// +/// # Safety +/// +/// The block must be fully initialized. Its incoming and outgoing edges, +/// if there are any, must point to initialized blocks, too. +/// +/// Note that the block might gain edges after this function returns, +/// as can happen during [gen_block_series]. Initialized here doesn't mean +/// ready to be consumed or that the machine code tracked by the block is +/// ready to be run. +/// +/// Due to this transient state where a block is tracked by the GC by +/// being inside an [IseqPayload] but not ready to be executed, it's +/// generally unsound to call any Ruby methods during codegen. That has +/// the potential to run blocks which are not ready. +unsafe fn add_block_version(blockref: BlockRef, cb: &CodeBlock) { + // SAFETY: caller ensures initialization + let block = unsafe { blockref.as_ref() }; // Function entry blocks must have stack size 0 - assert!(!(block.blockid.idx == 0 && block.ctx.stack_size > 0)); + assert!(!(block.iseq_range.start == 0 && block.ctx.stack_size > 0)); - let version_list = get_or_create_version_list(block.blockid); + let version_list = get_or_create_version_list(block.get_blockid()); - version_list.push(blockref.clone()); + version_list.push(blockref); version_list.shrink_to_fit(); // By writing the new block to the iseq, the iseq now // contains new references to Ruby objects. Run write barriers. - let iseq: VALUE = block.blockid.iseq.into(); - for &dep in block.iter_cme_deps() { + let iseq: VALUE = block.iseq.get().into(); + for dep in block.iter_cme_deps() { obj_written!(iseq, dep.into()); } @@ -1163,16 +1343,16 @@ fn add_block_version(blockref: &BlockRef, cb: &CodeBlock) { incr_counter!(compiled_block_count); // Mark code pages for code GC - let iseq_payload = get_iseq_payload(block.blockid.iseq).unwrap(); - for page in cb.addrs_to_pages(block.start_addr, block.end_addr.unwrap()) { + let iseq_payload = get_iseq_payload(block.iseq.get()).unwrap(); + for page in cb.addrs_to_pages(block.start_addr, block.end_addr.get()) { iseq_payload.pages.insert(page); } } /// Remove a block version from the version map of its parent ISEQ fn remove_block_version(blockref: &BlockRef) { - let block = blockref.borrow(); - let version_list = match get_version_list(block.blockid) { + let block = unsafe { blockref.as_ref() }; + let version_list = match get_version_list(block.get_blockid()) { Some(version_list) => version_list, None => return, }; @@ -1181,47 +1361,73 @@ fn remove_block_version(blockref: &BlockRef) { version_list.retain(|other| blockref != other); } -//=========================================================================== -// I put the implementation of traits for core.rs types below -// We can move these closer to the above structs later if we want. -//=========================================================================== +impl JITState { + // Finish compiling and turn a jit state into a block + // note that the block is still not in shape. + pub fn into_block(self, end_insn_idx: IseqIdx, start_addr: CodePtr, end_addr: CodePtr, gc_obj_offsets: Vec<u32>) -> BlockRef { + // Allocate the block and get its pointer + let blockref: *mut MaybeUninit<Block> = Box::into_raw(Box::new(MaybeUninit::uninit())); -impl Block { - pub fn new(blockid: BlockId, ctx: &Context, start_addr: CodePtr) -> BlockRef { - let block = Block { - blockid, - end_idx: 0, - ctx: ctx.clone(), + incr_counter_by!(num_gc_obj_refs, gc_obj_offsets.len()); + + // Make the new block + let block = MaybeUninit::new(Block { start_addr, - end_addr: None, - incoming: Vec::new(), - outgoing: Box::new([]), - gc_obj_offsets: Box::new([]), - cme_dependencies: Box::new([]), - entry_exit: None, - }; + iseq: Cell::new(self.get_iseq()), + iseq_range: self.get_starting_insn_idx()..end_insn_idx, + ctx: self.get_starting_ctx(), + end_addr: Cell::new(end_addr), + incoming: MutableBranchList(Cell::default()), + gc_obj_offsets: gc_obj_offsets.into_boxed_slice(), + entry_exit: self.get_block_entry_exit(), + cme_dependencies: self.method_lookup_assumptions.into_iter().map(Cell::new).collect(), + // Pending branches => actual branches + outgoing: self.pending_outgoing.into_iter().map(|pending_out| { + let pending_out = Rc::try_unwrap(pending_out) + .ok().expect("all PendingBranchRefs should be unique when ready to construct a Block"); + pending_out.into_branch(NonNull::new(blockref as *mut Block).expect("no null from Box")) + }).collect() + }); + // Initialize it on the heap + // SAFETY: allocated with Box above + unsafe { ptr::write(blockref, block) }; - // Wrap the block in a reference counted refcell - // so that the block ownership can be shared - BlockRef::new(Rc::new(RefCell::new(block))) - } + // Block is initialized now. Note that MaybeUnint<T> has the same layout as T. + let blockref = NonNull::new(blockref as *mut Block).expect("no null from Box"); - pub fn get_blockid(&self) -> BlockId { - self.blockid + // Track all the assumptions the block makes as invariants + if self.block_assumes_single_ractor { + track_single_ractor_assumption(blockref); + } + for bop in self.bop_assumptions { + track_bop_assumption(blockref, bop); + } + // SAFETY: just allocated it above + for cme in unsafe { blockref.as_ref() }.cme_dependencies.iter() { + track_method_lookup_stability_assumption(blockref, cme.get()); + } + if let Some(idlist) = self.stable_constant_names_assumption { + track_stable_constant_names_assumption(blockref, idlist); + } + + blockref } +} - pub fn get_end_idx(&self) -> u16 { - self.end_idx +impl Block { + pub fn get_blockid(&self) -> BlockId { + BlockId { iseq: self.iseq.get(), idx: self.iseq_range.start } } - pub fn get_ctx(&self) -> Context { - self.ctx.clone() + pub fn get_end_idx(&self) -> IseqIdx { + self.iseq_range.end } pub fn get_ctx_count(&self) -> usize { let mut count = 1; // block.ctx for branch in self.outgoing.iter() { - count += branch.borrow().get_stub_count(); + // SAFETY: &self implies it's initialized + count += unsafe { branch.as_ref() }.get_stub_count(); } count } @@ -1232,57 +1438,23 @@ impl Block { } #[allow(unused)] - pub fn get_end_addr(&self) -> Option<CodePtr> { - self.end_addr + pub fn get_end_addr(&self) -> CodePtr { + self.end_addr.get() } /// Get an immutable iterator over cme dependencies - pub fn iter_cme_deps(&self) -> std::slice::Iter<'_, CmePtr> { - self.cme_dependencies.iter() - } - - /// Set the end address in the generated for the block - /// This can be done only once for a block - pub fn set_end_addr(&mut self, addr: CodePtr) { - // TODO: assert constraint that blocks can shrink but not grow in length - self.end_addr = Some(addr); - } - - /// Set the index of the last instruction in the block - /// This can be done only once for a block - pub fn set_end_idx(&mut self, end_idx: u16) { - assert!(self.end_idx == 0); - self.end_idx = end_idx; - } - - pub fn set_gc_obj_offsets(self: &mut Block, gc_offsets: Vec<u32>) { - assert_eq!(self.gc_obj_offsets.len(), 0); - if !gc_offsets.is_empty() { - incr_counter_by!(num_gc_obj_refs, gc_offsets.len()); - self.gc_obj_offsets = gc_offsets.into_boxed_slice(); - } - } - - /// Instantiate a new CmeDependency struct and add it to the list of - /// dependencies for this block. - pub fn set_cme_dependencies(&mut self, cme_dependencies: Vec<CmePtr>) { - self.cme_dependencies = cme_dependencies.into_boxed_slice(); + pub fn iter_cme_deps(&self) -> impl Iterator<Item = CmePtr> + '_ { + self.cme_dependencies.iter().map(Cell::get) } // Push an incoming branch ref and shrink the vector - fn push_incoming(&mut self, branch: BranchRef) { + fn push_incoming(&self, branch: BranchRef) { self.incoming.push(branch); - self.incoming.shrink_to_fit(); - } - - // Push an outgoing branch ref and shrink the vector - pub fn set_outgoing(&mut self, outgoing: Vec<BranchRef>) { - self.outgoing = outgoing.into_boxed_slice(); } // Compute the size of the block code pub fn code_size(&self) -> usize { - (self.end_addr.unwrap().raw_ptr() as usize) - (self.start_addr.raw_ptr() as usize) + (self.end_addr.get().into_usize()) - (self.start_addr.into_usize()) } } @@ -1704,39 +1876,43 @@ fn gen_block_series_body( // Generate code for the first block let first_block = gen_single_block(blockid, start_ctx, ec, cb, ocb).ok()?; - batch.push(first_block.clone()); // Keep track of this block version + batch.push(first_block); // Keep track of this block version // Add the block version to the VersionMap for this ISEQ - add_block_version(&first_block, cb); + unsafe { add_block_version(first_block, cb) }; // Loop variable - let mut last_blockref = first_block.clone(); + let mut last_blockref = first_block; loop { // Get the last outgoing branch from the previous block. let last_branchref = { - let last_block = last_blockref.borrow(); + let last_block = unsafe { last_blockref.as_ref() }; match last_block.outgoing.last() { - Some(branch) => branch.clone(), + Some(branch) => *branch, None => { break; } // If last block has no branches, stop. } }; - let mut last_branch = last_branchref.borrow_mut(); + let last_branch = unsafe { last_branchref.as_ref() }; + + incr_counter!(block_next_count); // gen_direct_jump() can request a block to be placed immediately after by // leaving a single target that has a `None` address. - let last_target = match &mut last_branch.targets { - [Some(last_target), None] if last_target.get_address().is_none() => last_target, - _ => break + // SAFETY: no mutation inside the unsafe block + let (requested_blockid, requested_ctx) = unsafe { + match (last_branch.targets[0].ref_unchecked(), last_branch.targets[1].ref_unchecked()) { + (Some(last_target), None) if last_target.get_address().is_none() => { + (last_target.get_blockid(), last_target.get_ctx()) + } + _ => { + // We're done when no fallthrough block is requested + break; + } + } }; - incr_counter!(block_next_count); - - // Get id and context for the new block - let requested_blockid = last_target.get_blockid(); - let requested_ctx = last_target.get_ctx(); - // Generate new block using context from the last branch. let result = gen_single_block(requested_blockid, &requested_ctx, ec, cb, ocb); @@ -1744,10 +1920,10 @@ fn gen_block_series_body( if result.is_err() { // Remove previously compiled block // versions from the version map - mem::drop(last_branch); // end borrow - for blockref in &batch { - free_block(blockref); - remove_block_version(blockref); + for blockref in batch { + remove_block_version(&blockref); + // SAFETY: block was well connected because it was in a version_map + unsafe { free_block(blockref, false) }; } // Stop compiling @@ -1757,16 +1933,14 @@ fn gen_block_series_body( let new_blockref = result.unwrap(); // Add the block version to the VersionMap for this ISEQ - add_block_version(&new_blockref, cb); + unsafe { add_block_version(new_blockref, cb) }; // Connect the last branch and the new block - last_branch.targets[0] = Some(Box::new(BranchTarget::Block(new_blockref.clone()))); - new_blockref - .borrow_mut() - .push_incoming(last_branchref.clone()); + last_branch.targets[0].set(Some(Box::new(BranchTarget::Block(new_blockref)))); + unsafe { new_blockref.as_ref().incoming.push(last_branchref) }; // Track the block - batch.push(new_blockref.clone()); + batch.push(new_blockref); // Repeat with newest block last_blockref = new_blockref; @@ -1777,12 +1951,12 @@ fn gen_block_series_body( // If dump_iseq_disasm is active, see if this iseq's location matches the given substring. // If so, we print the new blocks to the console. if let Some(substr) = get_option_ref!(dump_iseq_disasm).as_ref() { - let blockid_idx = blockid.idx; - let iseq_location = iseq_get_location(blockid.iseq, blockid_idx); + let iseq_location = iseq_get_location(blockid.iseq, blockid.idx); if iseq_location.contains(substr) { - let last_block = last_blockref.borrow(); - println!("Compiling {} block(s) for {}, ISEQ offsets [{}, {})", batch.len(), iseq_location, blockid_idx, last_block.end_idx); - print!("{}", disasm_iseq_insn_range(blockid.iseq, blockid.idx, last_block.end_idx)); + let last_block = unsafe { last_blockref.as_ref() }; + let iseq_range = &last_block.iseq_range; + println!("Compiling {} block(s) for {}, ISEQ offsets [{}, {})", batch.len(), iseq_location, iseq_range.start, iseq_range.end); + print!("{}", disasm_iseq_insn_range(blockid.iseq, iseq_range.start, iseq_range.end)); } } } @@ -1829,8 +2003,8 @@ pub fn gen_entry_point(iseq: IseqPtr, ec: EcPtr) -> Option<CodePtr> { // If the block contains no Ruby instructions Some(block) => { - let block = block.borrow(); - if block.end_idx == insn_idx { + let block = unsafe { block.as_ref() }; + if block.iseq_range.is_empty() { return None; } } @@ -1841,13 +2015,14 @@ pub fn gen_entry_point(iseq: IseqPtr, ec: EcPtr) -> Option<CodePtr> { } /// Generate code for a branch, possibly rewriting and changing the size of it -fn regenerate_branch(cb: &mut CodeBlock, branch: &mut Branch) { +fn regenerate_branch(cb: &mut CodeBlock, branch: &Branch) { // Remove old comments - if let (Some(start_addr), Some(end_addr)) = (branch.start_addr, branch.end_addr) { - cb.remove_comments(start_addr, end_addr) - } + cb.remove_comments(branch.start_addr, branch.end_addr.get()); - let branch_terminates_block = branch.end_addr == branch.block.borrow().end_addr; + // SAFETY: having a &Branch implies branch.block is initialized. + let block = unsafe { branch.block.as_ref() }; + + let branch_terminates_block = branch.end_addr.get() == block.get_end_addr(); // Generate the branch let mut asm = Assembler::new(); @@ -1861,17 +2036,17 @@ fn regenerate_branch(cb: &mut CodeBlock, branch: &mut Branch) { // Rewrite the branch let old_write_pos = cb.get_write_pos(); let old_dropped_bytes = cb.has_dropped_bytes(); - cb.set_write_ptr(branch.start_addr.unwrap()); + cb.set_write_ptr(branch.start_addr); cb.set_dropped_bytes(false); asm.compile(cb); + let new_end_addr = cb.get_write_ptr(); - branch.end_addr = Some(cb.get_write_ptr()); + branch.end_addr.set(new_end_addr); // The block may have shrunk after the branch is rewritten - let mut block = branch.block.borrow_mut(); if branch_terminates_block { // Adjust block size - block.end_addr = branch.end_addr; + block.end_addr.set(new_end_addr); } // cb.write_pos is both a write cursor and a marker for the end of @@ -1891,34 +2066,29 @@ fn regenerate_branch(cb: &mut CodeBlock, branch: &mut Branch) { } } -/// Create a new outgoing branch entry for a block -fn make_branch_entry(jit: &mut JITState, block: &BlockRef, gen_fn: BranchGenFn) -> BranchRef { - let branch = Branch { - // Block this is attached to - block: block.clone(), - - // Positions where the generated code starts and ends - start_addr: None, - end_addr: None, +pub type PendingBranchRef = Rc<PendingBranch>; - // Branch target blocks and their contexts - targets: [None, None], - - // Branch code generation function +/// Create a new outgoing branch entry for a block +fn new_pending_branch(jit: &mut JITState, gen_fn: BranchGenFn) -> PendingBranchRef { + let branch = Rc::new(PendingBranch { + uninit_branch: Box::new(MaybeUninit::uninit()), gen_fn, - }; + start_addr: Cell::new(None), + end_addr: Cell::new(None), + targets: [Cell::new(None), Cell::new(None)], + }); + + incr_counter!(compiled_branch_count); // TODO not true. count at finalize time // Add to the list of outgoing branches for the block - let branchref = Rc::new(RefCell::new(branch)); - jit.push_outgoing(branchref.clone()); - incr_counter!(compiled_branch_count); + jit.queue_outgoing_branch(branch.clone()); - return branchref; + branch } c_callable! { /// Generated code calls this function with the SysV calling convention. - /// See [set_branch_target]. + /// See [gen_call_branch_stub_hit]. fn branch_stub_hit( branch_ptr: *const c_void, target_idx: u32, @@ -1937,39 +2107,38 @@ fn branch_stub_hit_body(branch_ptr: *const c_void, target_idx: u32, ec: EcPtr) - println!("branch_stub_hit"); } - assert!(!branch_ptr.is_null()); - - //branch_ptr is actually: - //branch_ptr: *const RefCell<Branch> - let branch_rc = unsafe { BranchRef::from_raw(branch_ptr as *const RefCell<Branch>) }; - - // We increment the strong count because we want to keep the reference owned - // by the branch stub alive. Return branch stubs can be hit multiple times. - unsafe { Rc::increment_strong_count(branch_ptr) }; + let branch_ref = NonNull::<Branch>::new(branch_ptr as *mut Branch) + .expect("Branches should not be null"); - let mut branch = branch_rc.borrow_mut(); + // SAFETY: We have the VM lock, and the branch is initialized by the time generated + // code calls this function. + let branch = unsafe { branch_ref.as_ref() }; let branch_size_on_entry = branch.code_size(); + let housing_block = unsafe { branch.block.as_ref() }; let target_idx: usize = target_idx.as_usize(); - let target = branch.targets[target_idx].as_ref().unwrap(); - let target_blockid = target.get_blockid(); - let target_ctx = target.get_ctx(); - let target_branch_shape = match target_idx { 0 => BranchShape::Next0, 1 => BranchShape::Next1, _ => unreachable!("target_idx < 2 must always hold"), }; + let (target_blockid, target_ctx): (BlockId, Context) = unsafe { + // SAFETY: no mutation of the target's Cell. Just reading out data. + let target = branch.targets[target_idx].ref_unchecked().as_ref().unwrap(); + + // If this branch has already been patched, return the dst address + // Note: recursion can cause the same stub to be hit multiple times + if let BranchTarget::Block(_) = target.as_ref() { + return target.get_address().unwrap().raw_ptr(); + } + + (target.get_blockid(), target.get_ctx()) + }; + let cb = CodegenGlobals::get_inline_cb(); let ocb = CodegenGlobals::get_outlined_cb(); - // If this branch has already been patched, return the dst address - // Note: ractors can cause the same stub to be hit multiple times - if let BranchTarget::Block(_) = target.as_ref() { - return target.get_address().unwrap().raw_ptr(); - } - let (cfp, original_interp_sp) = unsafe { let cfp = get_ec_cfp(ec); let original_interp_sp = get_cfp_sp(cfp); @@ -1996,78 +2165,70 @@ fn branch_stub_hit_body(branch_ptr: *const c_void, target_idx: u32, ec: EcPtr) - // Try to find an existing compiled version of this block let mut block = find_block_version(target_blockid, &target_ctx); - + let mut branch_modified = false; // If this block hasn't yet been compiled if block.is_none() { let branch_old_shape = branch.gen_fn.get_shape(); - let mut branch_modified = false; + // If the new block can be generated right after the branch (at cb->write_pos) - if Some(cb.get_write_ptr()) == branch.end_addr { + if cb.get_write_ptr() == branch.end_addr.get() { // This branch should be terminating its block - assert!(branch.end_addr == branch.block.borrow().end_addr); + assert!(branch.end_addr == housing_block.end_addr); // Change the branch shape to indicate the target block will be placed next branch.gen_fn.set_shape(target_branch_shape); // Rewrite the branch with the new, potentially more compact shape - regenerate_branch(cb, &mut branch); + regenerate_branch(cb, branch); branch_modified = true; // Ensure that the branch terminates the codeblock just like // before entering this if block. This drops bytes off the end // in case we shrank the branch when regenerating. - cb.set_write_ptr(branch.end_addr.unwrap()); + cb.set_write_ptr(branch.end_addr.get()); } // Compile the new block version - drop(branch); // Stop mutable RefCell borrow since GC might borrow branch for marking block = gen_block_series(target_blockid, &target_ctx, ec, cb, ocb); - branch = branch_rc.borrow_mut(); if block.is_none() && branch_modified { // We couldn't generate a new block for the branch, but we modified the branch. // Restore the branch by regenerating it. branch.gen_fn.set_shape(branch_old_shape); - regenerate_branch(cb, &mut branch); + regenerate_branch(cb, branch); } } // Finish building the new block let dst_addr = match block { - Some(block_rc) => { - let mut block: RefMut<_> = block_rc.borrow_mut(); + Some(new_block) => { + let new_block = unsafe { new_block.as_ref() }; // Branch shape should reflect layout - assert!(!(branch.gen_fn.get_shape() == target_branch_shape && Some(block.start_addr) != branch.end_addr)); + assert!(!(branch.gen_fn.get_shape() == target_branch_shape && new_block.start_addr != branch.end_addr.get())); // Add this branch to the list of incoming branches for the target - block.push_incoming(branch_rc.clone()); - mem::drop(block); // end mut borrow + new_block.push_incoming(branch_ref); // Update the branch target address - branch.targets[target_idx] = Some(Box::new(BranchTarget::Block(block_rc.clone()))); + branch.targets[target_idx].set(Some(Box::new(BranchTarget::Block(new_block.into())))); // Rewrite the branch with the new jump target address - regenerate_branch(cb, &mut branch); + regenerate_branch(cb, branch); // Restore interpreter sp, since the code hitting the stub expects the original. unsafe { rb_set_cfp_sp(cfp, original_interp_sp) }; - block_rc.borrow().start_addr + new_block.start_addr } None => { - // Code GC needs to borrow blocks for invalidation, so their mutable - // borrows must be dropped first. - drop(block); - drop(branch); // Trigger code GC. The whole ISEQ will be recompiled later. // We shouldn't trigger it in the middle of compilation in branch_stub_hit // because incomplete code could be used when cb.dropped_bytes is flipped // by code GC. So this place, after all compilation, is the safest place // to hook code GC on branch_stub_hit. cb.code_gc(); - branch = branch_rc.borrow_mut(); // Failed to service the stub by generating a new block so now we // need to exit to the interpreter at the stubbed location. We are @@ -2087,55 +2248,35 @@ fn branch_stub_hit_body(branch_ptr: *const c_void, target_idx: u32, ec: EcPtr) - assert!( new_branch_size <= branch_size_on_entry, "branch stubs should never enlarge branches (start_addr: {:?}, old_size: {}, new_size: {})", - branch.start_addr.unwrap().raw_ptr(), branch_size_on_entry, new_branch_size, + branch.start_addr.raw_ptr(), branch_size_on_entry, new_branch_size, ); // Return a pointer to the compiled block version dst_addr.raw_ptr() } -/// Set up a branch target at an index with a block version or a stub -fn set_branch_target( - target_idx: u32, - target: BlockId, - ctx: &Context, - branchref: &BranchRef, - branch: &mut Branch, +/// Generate a "stub", a piece of code that calls the compiler back when run. +/// A piece of code that redeems for more code; a thunk for code. +fn gen_call_branch_stub_hit( ocb: &mut OutlinedCb, -) { - let maybe_block = find_block_version(target, ctx); - - // If the block already exists - if let Some(blockref) = maybe_block { - let mut block = blockref.borrow_mut(); - - // Add an incoming branch into this block - block.push_incoming(branchref.clone()); - - // Fill out the target with this block - branch.targets[target_idx.as_usize()] = Some(Box::new(BranchTarget::Block(blockref.clone()))); - - return; - } - + branch_struct_address: usize, + target_idx: u32, +) -> Option<CodePtr> { let ocb = ocb.unwrap(); // Generate an outlined stub that will call branch_stub_hit() let stub_addr = ocb.get_write_ptr(); - // Get a raw pointer to the branch. We clone and then decrement the strong count which overall - // balances the strong count. We do this so that we're passing the result of [Rc::into_raw] to - // [Rc::from_raw] as required. - // We make sure the block housing the branch is still alive when branch_stub_hit() is running. - let branch_ptr: *const RefCell<Branch> = BranchRef::into_raw(branchref.clone()); - unsafe { BranchRef::decrement_strong_count(branch_ptr) }; - let mut asm = Assembler::new(); asm.comment("branch stub hit"); // Set up the arguments unique to this stub for: - // branch_stub_hit(branch_ptr, target_idx, ec) - asm.mov(C_ARG_OPNDS[0], Opnd::const_ptr(branch_ptr as *const u8)); + // + // branch_stub_hit(branch_ptr, target_idx, ec) + // + // Bake pointer to Branch into output code. + // We make sure the block housing the branch is still alive when branch_stub_hit() is running. + asm.mov(C_ARG_OPNDS[0], branch_struct_address.into()); asm.mov(C_ARG_OPNDS[1], target_idx.into()); // Jump to trampoline to call branch_stub_hit() @@ -2146,13 +2287,9 @@ fn set_branch_target( if ocb.has_dropped_bytes() { // No space + None } else { - // Fill the branch target with a stub - branch.targets[target_idx.as_usize()] = Some(Box::new(BranchTarget::Stub(Box::new(BranchStub { - address: Some(stub_addr), - id: target, - ctx: ctx.clone(), - })))); + Some(stub_addr) } } @@ -2188,28 +2325,26 @@ pub fn gen_branch_stub_hit_trampoline(ocb: &mut OutlinedCb) -> CodePtr { impl Assembler { // Mark the start position of a patchable branch in the machine code - fn mark_branch_start(&mut self, branchref: &BranchRef) + fn mark_branch_start(&mut self, branchref: &PendingBranchRef) { // We need to create our own branch rc object // so that we can move the closure below let branchref = branchref.clone(); self.pos_marker(move |code_ptr| { - let mut branch = branchref.borrow_mut(); - branch.start_addr = Some(code_ptr); + branchref.start_addr.set(Some(code_ptr)); }); } // Mark the end position of a patchable branch in the machine code - fn mark_branch_end(&mut self, branchref: &BranchRef) + fn mark_branch_end(&mut self, branchref: &PendingBranchRef) { // We need to create our own branch rc object // so that we can move the closure below let branchref = branchref.clone(); self.pos_marker(move |code_ptr| { - let mut branch = branchref.borrow_mut(); - branch.end_addr = Some(code_ptr); + branchref.end_addr.set(Some(code_ptr)); }); } } @@ -2224,66 +2359,63 @@ pub fn gen_branch( ctx1: Option<&Context>, gen_fn: BranchGenFn, ) { - let branchref = make_branch_entry(jit, &jit.get_block(), gen_fn); - let branch = &mut branchref.borrow_mut(); + let branch = new_pending_branch(jit, gen_fn); // Get the branch targets or stubs - set_branch_target(0, target0, ctx0, &branchref, branch, ocb); - if let Some(ctx) = ctx1 { - set_branch_target(1, target1.unwrap(), ctx, &branchref, branch, ocb); - if branch.targets[1].is_none() { - return; // avoid unwrap() in gen_fn() + let target0_addr = branch.set_target(0, target0, ctx0, ocb); + let target1_addr = if let Some(ctx) = ctx1 { + let addr = branch.set_target(1, target1.unwrap(), ctx, ocb); + if addr.is_none() { + // target1 requested but we're out of memory. + // Avoid unwrap() in gen_fn() + return; } - } + + addr + } else { None }; // Call the branch generation function - asm.mark_branch_start(&branchref); - if let Some(dst_addr) = branch.get_target_address(0) { - gen_fn.call(asm, dst_addr, branch.get_target_address(1)); + asm.mark_branch_start(&branch); + if let Some(dst_addr) = target0_addr { + branch.gen_fn.call(asm, dst_addr, target1_addr); } - asm.mark_branch_end(&branchref); + asm.mark_branch_end(&branch); } pub fn gen_direct_jump(jit: &mut JITState, ctx: &Context, target0: BlockId, asm: &mut Assembler) { - let branchref = make_branch_entry(jit, &jit.get_block(), BranchGenFn::JumpToTarget0(BranchShape::Default)); - let mut branch = branchref.borrow_mut(); - - let mut new_target = BranchTarget::Stub(Box::new(BranchStub { - address: None, - ctx: ctx.clone(), - id: target0, - })); - + let branch = new_pending_branch(jit, BranchGenFn::JumpToTarget0(Cell::new(BranchShape::Default))); let maybe_block = find_block_version(target0, ctx); // If the block already exists - if let Some(blockref) = maybe_block { - let mut block = blockref.borrow_mut(); + let new_target = if let Some(blockref) = maybe_block { + let block = unsafe { blockref.as_ref() }; let block_addr = block.start_addr; - block.push_incoming(branchref.clone()); - - new_target = BranchTarget::Block(blockref.clone()); - - branch.gen_fn.set_shape(BranchShape::Default); - // Call the branch generation function asm.comment("gen_direct_jmp: existing block"); - asm.mark_branch_start(&branchref); + asm.mark_branch_start(&branch); branch.gen_fn.call(asm, block_addr, None); - asm.mark_branch_end(&branchref); - } else { - // `None` in new_target.address signals gen_block_series() to compile the - // target block right after this one (fallthrough). - branch.gen_fn.set_shape(BranchShape::Next0); + asm.mark_branch_end(&branch); + BranchTarget::Block(blockref) + } else { // The branch is effectively empty (a noop) asm.comment("gen_direct_jmp: fallthrough"); - asm.mark_branch_start(&branchref); - asm.mark_branch_end(&branchref); - } + asm.mark_branch_start(&branch); + asm.mark_branch_end(&branch); + branch.gen_fn.set_shape(BranchShape::Next0); + + // `None` in new_target.address signals gen_block_series() to + // compile the target block right after this one (fallthrough). + BranchTarget::Stub(Box::new(BranchStub { + address: None, + ctx: ctx.clone(), + iseq: Cell::new(target0.iseq), + iseq_idx: target0.idx, + })) + }; - branch.targets[0] = Some(Box::new(new_target)); + branch.targets[0].set(Some(Box::new(new_target))); } /// Create a stub to force the code up to this point to be executed @@ -2304,81 +2436,127 @@ pub fn defer_compilation( } next_ctx.chain_depth += 1; - let block_rc = jit.get_block(); - let branch_rc = make_branch_entry(jit, &jit.get_block(), BranchGenFn::JumpToTarget0(BranchShape::Default)); - let mut branch = branch_rc.borrow_mut(); - let block = block_rc.borrow(); + let branch = new_pending_branch(jit, BranchGenFn::JumpToTarget0(Cell::new(BranchShape::Default))); let blockid = BlockId { - iseq: block.blockid.iseq, + iseq: jit.get_iseq(), idx: jit.get_insn_idx(), }; - set_branch_target(0, blockid, &next_ctx, &branch_rc, &mut branch, ocb); + + // Likely a stub due to the increased chain depth + let target0_address = branch.set_target(0, blockid, &next_ctx, ocb); // Call the branch generation function asm.comment("defer_compilation"); - asm.mark_branch_start(&branch_rc); - if let Some(dst_addr) = branch.get_target_address(0) { + asm.mark_branch_start(&branch); + if let Some(dst_addr) = target0_address { branch.gen_fn.call(asm, dst_addr, None); } - asm.mark_branch_end(&branch_rc); + asm.mark_branch_end(&branch); // If the block we're deferring from is empty - if jit.get_block().borrow().get_blockid().idx == jit.get_insn_idx() { + if jit.get_starting_insn_idx() == jit.get_insn_idx() { incr_counter!(defer_empty_count); } incr_counter!(defer_count); } -fn remove_from_graph(blockref: &BlockRef) { - let block = blockref.borrow(); +/// Remove a block from the live control flow graph. +/// Block must be initialized and incoming/outgoing edges +/// must also point to initialized blocks. +unsafe fn remove_from_graph(blockref: BlockRef) { + let block = unsafe { blockref.as_ref() }; // Remove this block from the predecessor's targets - for pred_branchref in &block.incoming { + for pred_branchref in block.incoming.0.take().iter() { // Branch from the predecessor to us - let mut pred_branch = pred_branchref.borrow_mut(); + let pred_branch = unsafe { pred_branchref.as_ref() }; // If this is us, nullify the target block - for target_idx in 0..=1 { - if let Some(target) = pred_branch.targets[target_idx].as_ref() { - if target.get_block().as_ref() == Some(blockref) { - pred_branch.targets[target_idx] = None; - } + for target_idx in 0..pred_branch.targets.len() { + // SAFETY: no mutation inside unsafe + let target_is_us = unsafe { + pred_branch.targets[target_idx] + .ref_unchecked() + .as_ref() + .and_then(|target| target.get_block()) + .and_then(|target_block| (target_block == blockref).then(|| ())) + .is_some() + }; + + if target_is_us { + pred_branch.targets[target_idx].set(None); } } } // For each outgoing branch for out_branchref in block.outgoing.iter() { - let out_branch = out_branchref.borrow(); - + let out_branch = unsafe { out_branchref.as_ref() }; // For each successor block - for out_target in out_branch.targets.iter().flatten() { - if let Some(succ_blockref) = &out_target.get_block() { + for out_target in out_branch.targets.iter() { + // SAFETY: copying out an Option<BlockRef>. No mutation. + let succ_block: Option<BlockRef> = unsafe { + out_target.ref_unchecked().as_ref().and_then(|target| target.get_block()) + }; + + if let Some(succ_block) = succ_block { // Remove outgoing branch from the successor's incoming list - let mut succ_block = succ_blockref.borrow_mut(); - succ_block - .incoming - .retain(|succ_incoming| !Rc::ptr_eq(succ_incoming, out_branchref)); + // SAFETY: caller promises the block has valid outgoing edges. + let succ_block = unsafe { succ_block.as_ref() }; + // Temporarily move out of succ_block.incoming. + let succ_incoming = succ_block.incoming.0.take(); + let mut succ_incoming = succ_incoming.into_vec(); + succ_incoming.retain(|branch| branch != out_branchref); + succ_block.incoming.0.set(succ_incoming.into_boxed_slice()); // allocs. Rely on oom=abort } } } } -/// Remove most references to a block to deallocate it. -/// Does not touch references from iseq payloads. -pub fn free_block(blockref: &BlockRef) { - block_assumptions_free(blockref); +/// Tear down a block and deallocate it. +/// Caller has to ensure that the code tracked by the block is not +/// running, as running code may hit [branch_stub_hit] who exepcts +/// [Branch] to be live. +/// +/// We currently ensure this through the `jit_cont` system in cont.c +/// and sometimes through the GC calling [rb_yjit_iseq_free]. The GC +/// has proven that an ISeq is not running if it calls us to free it. +/// +/// For delayed deallocation, since dead blocks don't keep +/// blocks they refer alive, by the time we get here their outgoing +/// edges may be dangling. Pass `graph_intact=false` such these cases. +pub unsafe fn free_block(blockref: BlockRef, graph_intact: bool) { + // Careful with order here. + // First, remove all pointers to the referent block + unsafe { + block_assumptions_free(blockref); + + if graph_intact { + remove_from_graph(blockref); + } + } - remove_from_graph(blockref); + // SAFETY: we should now have a unique pointer to the block + unsafe { dealloc_block(blockref) } +} - // Branches have a Rc pointing at the block housing them. - // Break the cycle. - blockref.borrow_mut().incoming.clear(); - blockref.borrow_mut().outgoing = Box::new([]); +/// Deallocate a block and its outgoing branches. Blocks own their outgoing branches. +/// Caller must ensure that we have unique ownership for the referent block +unsafe fn dealloc_block(blockref: BlockRef) { + unsafe { + for outgoing in blockref.as_ref().outgoing.iter() { + // this Box::from_raw matches the Box::into_raw from PendingBranch::into_branch + mem::drop(Box::from_raw(outgoing.as_ptr())); + } + } - // No explicit deallocation here as blocks are ref-counted. + // Deallocate the referent Block + unsafe { + // this Box::from_raw matches the Box::into_raw from JITState::into_block + mem::drop(Box::from_raw(blockref.as_ptr())); + } } // Some runtime checks for integrity of a program location @@ -2396,20 +2574,21 @@ pub fn invalidate_block_version(blockref: &BlockRef) { // TODO: want to assert that all other ractors are stopped here. Can't patch // machine code that some other thread is running. - let block = blockref.borrow(); + let block = unsafe { (*blockref).as_ref() }; + let id_being_invalidated = block.get_blockid(); let mut cb = CodegenGlobals::get_inline_cb(); let ocb = CodegenGlobals::get_outlined_cb(); - verify_blockid(block.blockid); + verify_blockid(id_being_invalidated); #[cfg(feature = "disasm")] { // If dump_iseq_disasm is specified, print to console that blocks for matching ISEQ names were invalidated. if let Some(substr) = get_option_ref!(dump_iseq_disasm).as_ref() { - let blockid_idx = block.blockid.idx; - let iseq_location = iseq_get_location(block.blockid.iseq, blockid_idx); + let iseq_range = &block.iseq_range; + let iseq_location = iseq_get_location(block.iseq.get(), iseq_range.start); if iseq_location.contains(substr) { - println!("Invalidating block from {}, ISEQ offsets [{}, {})", iseq_location, blockid_idx, block.end_idx); + println!("Invalidating block from {}, ISEQ offsets [{}, {})", iseq_location, iseq_range.start, iseq_range.end); } } } @@ -2431,9 +2610,7 @@ pub fn invalidate_block_version(blockref: &BlockRef) { .entry_exit .expect("invalidation needs the entry_exit field"); { - let block_end = block - .end_addr - .expect("invalidation needs constructed block"); + let block_end = block.get_end_addr(); if block_start == block_entry_exit { // Some blocks exit on entry. Patching a jump to the entry at the @@ -2462,10 +2639,8 @@ pub fn invalidate_block_version(blockref: &BlockRef) { } // For each incoming branch - mem::drop(block); // end borrow: regenerate_branch might mut borrow this - let block = blockref.borrow().clone(); - for branchref in &block.incoming { - let mut branch = branchref.borrow_mut(); + for branchref in block.incoming.0.take().iter() { + let branch = unsafe { branchref.as_ref() }; let target_idx = if branch.get_target_address(0) == Some(block_start) { 0 } else { @@ -2473,30 +2648,35 @@ pub fn invalidate_block_version(blockref: &BlockRef) { }; // Assert that the incoming branch indeed points to the block being invalidated - let incoming_target = branch.targets[target_idx].as_ref().unwrap(); - assert_eq!(Some(block_start), incoming_target.get_address()); - if let Some(incoming_block) = &incoming_target.get_block() { - assert_eq!(blockref, incoming_block); + // SAFETY: no mutation. + unsafe { + let incoming_target = branch.targets[target_idx].ref_unchecked().as_ref().unwrap(); + assert_eq!(Some(block_start), incoming_target.get_address()); + if let Some(incoming_block) = &incoming_target.get_block() { + assert_eq!(blockref, incoming_block); + } } - // Create a stub for this branch target or rewire it to a valid block - set_branch_target(target_idx as u32, block.blockid, &block.ctx, branchref, &mut branch, ocb); + // Create a stub for this branch target + let stub_addr = gen_call_branch_stub_hit(ocb, branchref.as_ptr() as usize, target_idx as u32); - if branch.targets[target_idx].is_none() { - // We were unable to generate a stub (e.g. OOM). Use the block's - // exit instead of a stub for the block. It's important that we - // still patch the branch in this situation so stubs are unique - // to branches. Think about what could go wrong if we run out of - // memory in the middle of this loop. - branch.targets[target_idx] = Some(Box::new(BranchTarget::Stub(Box::new(BranchStub { - address: block.entry_exit, - id: block.blockid, - ctx: block.ctx.clone(), - })))); - } + // In case we were unable to generate a stub (e.g. OOM). Use the block's + // exit instead of a stub for the block. It's important that we + // still patch the branch in this situation so stubs are unique + // to branches. Think about what could go wrong if we run out of + // memory in the middle of this loop. + let stub_addr = stub_addr.unwrap_or(block_entry_exit); + + // Fill the branch target with a stub + branch.targets[target_idx].set(Some(Box::new(BranchTarget::Stub(Box::new(BranchStub { + address: Some(stub_addr), + iseq: block.iseq.clone(), + iseq_idx: block.iseq_range.start, + ctx: block.ctx.clone(), + }))))); // Check if the invalidated block immediately follows - let target_next = Some(block.start_addr) == branch.end_addr; + let target_next = block.start_addr == branch.end_addr.get(); if target_next { // The new block will no longer be adjacent. @@ -2507,7 +2687,7 @@ pub fn invalidate_block_version(blockref: &BlockRef) { // Rewrite the branch with the new jump target address let old_branch_size = branch.code_size(); - regenerate_branch(cb, &mut branch); + regenerate_branch(cb, branch); if target_next && branch.end_addr > block.end_addr { panic!("yjit invalidate rewrote branch past end of invalidated block: {:?} (code_size: {})", branch, block.code_size()); @@ -2515,7 +2695,7 @@ pub fn invalidate_block_version(blockref: &BlockRef) { if !target_next && branch.code_size() > old_branch_size { panic!( "invalidated branch grew in size (start_addr: {:?}, old_size: {}, new_size: {})", - branch.start_addr.unwrap().raw_ptr(), old_branch_size, branch.code_size() + branch.start_addr.raw_ptr(), old_branch_size, branch.code_size() ); } } @@ -2528,17 +2708,21 @@ pub fn invalidate_block_version(blockref: &BlockRef) { // points will always have an instruction index of 0. We'll need to // change this in the future when we support optional parameters because // they enter the function with a non-zero PC - if block.blockid.idx == 0 { + if block.iseq_range.start == 0 { // TODO: // We could reset the exec counter to zero in rb_iseq_reset_jit_func() // so that we eventually compile a new entry point when useful - unsafe { rb_iseq_reset_jit_func(block.blockid.iseq) }; + unsafe { rb_iseq_reset_jit_func(block.iseq.get()) }; } // FIXME: // Call continuation addresses on the stack can also be atomically replaced by jumps going to the stub. - delayed_deallocation(blockref); + // SAFETY: This block was in a version_map earlier + // in this function before we removed it, so it's well connected. + unsafe { remove_from_graph(*blockref) }; + + delayed_deallocation(*blockref); ocb.unwrap().mark_all_executable(); cb.mark_all_executable(); @@ -2553,22 +2737,49 @@ pub fn invalidate_block_version(blockref: &BlockRef) { // invalidated branch pointers. Example: // def foo(n) // if n == 2 +// # 1.times{} to use a cfunc to avoid exiting from the +// # frame which will use the retained return address // return 1.times { Object.define_method(:foo) {} } // end // // foo(n + 1) // end // p foo(1) -pub fn delayed_deallocation(blockref: &BlockRef) { +pub fn delayed_deallocation(blockref: BlockRef) { block_assumptions_free(blockref); - // We do this another time when we deem that it's safe - // to deallocate in case there is another Ractor waiting to acquire the - // VM lock inside branch_stub_hit(). - remove_from_graph(blockref); + let payload = get_iseq_payload(unsafe { blockref.as_ref() }.iseq.get()).unwrap(); + payload.dead_blocks.push(blockref); +} + +trait RefUnchecked { + type Contained; + unsafe fn ref_unchecked(&self) -> &Self::Contained; +} + +impl<T> RefUnchecked for Cell<T> { + type Contained = T; - let payload = get_iseq_payload(blockref.borrow().blockid.iseq).unwrap(); - payload.dead_blocks.push(blockref.clone()); + /// Gives a reference to the contents of a [Cell]. + /// Dangerous; please include a SAFETY note. + /// + /// An easy way to use this without triggering Undefined Behavior is to + /// 1. ensure there is transitively no Cell/UnsafeCell mutation in the `unsafe` block + /// 2. ensure the `unsafe` block does not return any references, so our + /// analysis is lexically confined. This is trivially true if the block + /// returns a `bool`, for example. Aggregates that store references have + /// explicit lifetime parameters that look like `<'a>`. + /// + /// There are other subtler situations that don't follow these rules yet + /// are still sound. + /// See `test_miri_ref_unchecked()` for examples. You can play with it + /// with `cargo +nightly miri test miri`. + unsafe fn ref_unchecked(&self) -> &Self::Contained { + // SAFETY: pointer is dereferenceable because it's from a &Cell. + // It's up to the caller to follow aliasing rules with the output + // reference. + unsafe { self.as_ptr().as_ref().unwrap() } + } } #[cfg(test)] @@ -2603,4 +2814,97 @@ mod tests { // TODO: write more tests for Context type diff } + + #[test] + fn test_miri_ref_unchecked() { + let blockid = BlockId { + iseq: ptr::null(), + idx: 0, + }; + let cb = CodeBlock::new_dummy(1024); + let dumm_addr = cb.get_write_ptr(); + let block = JITState::new(blockid, Context::default(), dumm_addr, ptr::null()) + .into_block(0, dumm_addr, dumm_addr, vec![]); + let _dropper = BlockDropper(block); + + // Outside of brief moments during construction, + // we're always working with &Branch (a shared reference to a Branch). + let branch: &Branch = &Branch { + gen_fn: BranchGenFn::JZToTarget0, + block, + start_addr: dumm_addr, + end_addr: Cell::new(dumm_addr), + targets: [Cell::new(None), Cell::new(Some(Box::new(BranchTarget::Stub(Box::new(BranchStub { + iseq: Cell::new(ptr::null()), + iseq_idx: 0, + address: None, + ctx: Context::default(), + })))))] + }; + // For easier soundness reasoning, make sure the reference returned does not out live the + // `unsafe` block! It's tempting to do, but it leads to non-local issues. + // Here is an example where it goes wrong: + if false { + for target in branch.targets.iter().as_ref() { + if let Some(btarget) = unsafe { target.ref_unchecked() } { + // btarget is derived from the usnafe block! + target.set(None); // This drops the contents of the cell... + assert!(btarget.get_address().is_none()); // but `btarget` is still live! UB. + } + } + } + + // Do something like this instead. It's not pretty, but it's easier to vet for UB this way. + for target in branch.targets.iter().as_ref() { + // SAFETY: no mutation within unsafe + if unsafe { target.ref_unchecked().is_none() } { + continue; + } + // SAFETY: no mutation within unsafe + assert!(unsafe { target.ref_unchecked().as_ref().unwrap().get_address().is_none() }); + target.set(None); + } + + // A more subtle situation where we do Cell/UnsafeCell mutation over the + // lifetime of the reference released by ref_unchecked(). + branch.targets[0].set(Some(Box::new(BranchTarget::Stub(Box::new(BranchStub { + iseq: Cell::new(ptr::null()), + iseq_idx: 0, + address: None, + ctx: Context::default(), + }))))); + // Invalid ISeq; we never dereference it. + let secret_iseq = NonNull::<rb_iseq_t>::dangling().as_ptr(); + unsafe { + if let Some(branch_target) = branch.targets[0].ref_unchecked().as_ref() { + if let BranchTarget::Stub(stub) = branch_target.as_ref() { + // SAFETY: + // This is a Cell mutation, but it mutates the contents + // of a a Cell<IseqPtr>, which is a different type + // from the type of Cell found in `Branch::targets`, so + // there is no chance of mutating the Cell that we called + // ref_unchecked() on above. + Cell::set(&stub.iseq, secret_iseq); + } + } + }; + // Check that we indeed changed the iseq of the stub + // Cell::take moves out of the cell. + assert_eq!( + secret_iseq as usize, + branch.targets[0].take().unwrap().get_blockid().iseq as usize + ); + + struct BlockDropper(BlockRef); + impl Drop for BlockDropper { + fn drop(&mut self) { + // SAFETY: we have ownership because the test doesn't stash + // the block away in any global structure. + // Note that the test being self-contained is also why we + // use dealloc_block() over free_block(), as free_block() touches + // the global invariants tables unavailable in tests. + unsafe { dealloc_block(self.0) }; + } + } + } } diff --git a/yjit/src/disasm.rs b/yjit/src/disasm.rs index 0b464b9333..f9a5744979 100644 --- a/yjit/src/disasm.rs +++ b/yjit/src/disasm.rs @@ -37,18 +37,23 @@ pub extern "C" fn rb_yjit_disasm_iseq(_ec: EcPtr, _ruby_self: VALUE, iseqw: VALU // This will truncate disassembly of methods with 10k+ bytecodes. // That's a good thing - this prints to console. - let out_string = disasm_iseq_insn_range(iseq, 0, 9999); + let out_string = with_vm_lock(src_loc!(), || disasm_iseq_insn_range(iseq, 0, 9999)); return rust_str_to_ruby(&out_string); } } +/// Only call while holding the VM lock. #[cfg(feature = "disasm")] pub fn disasm_iseq_insn_range(iseq: IseqPtr, start_idx: u16, end_idx: u16) -> String { let mut out = String::from(""); // Get a list of block versions generated for this iseq - let mut block_list = get_or_create_iseq_block_list(iseq); + let block_list = get_or_create_iseq_block_list(iseq); + let mut block_list: Vec<&Block> = block_list.into_iter().map(|blockref| { + // SAFETY: We have the VM lock here and all the blocks on iseqs are valid. + unsafe { blockref.as_ref() } + }).collect(); // Get a list of codeblocks relevant to this iseq let global_cb = crate::codegen::CodegenGlobals::get_inline_cb(); @@ -58,8 +63,8 @@ pub fn disasm_iseq_insn_range(iseq: IseqPtr, start_idx: u16, end_idx: u16) -> St use std::cmp::Ordering; // Get the start addresses for each block - let addr_a = a.borrow().get_start_addr().raw_ptr(); - let addr_b = b.borrow().get_start_addr().raw_ptr(); + let addr_a = a.get_start_addr().raw_ptr(); + let addr_b = b.get_start_addr().raw_ptr(); if addr_a < addr_b { Ordering::Less @@ -73,20 +78,19 @@ pub fn disasm_iseq_insn_range(iseq: IseqPtr, start_idx: u16, end_idx: u16) -> St // Compute total code size in bytes for all blocks in the function let mut total_code_size = 0; for blockref in &block_list { - total_code_size += blockref.borrow().code_size(); + total_code_size += blockref.code_size(); } writeln!(out, "NUM BLOCK VERSIONS: {}", block_list.len()).unwrap(); writeln!(out, "TOTAL INLINE CODE SIZE: {} bytes", total_code_size).unwrap(); // For each block, sorted by increasing start address - for block_idx in 0..block_list.len() { - let block = block_list[block_idx].borrow(); + for (block_idx, block) in block_list.iter().enumerate() { let blockid = block.get_blockid(); if blockid.idx >= start_idx && blockid.idx < end_idx { let end_idx = block.get_end_idx(); let start_addr = block.get_start_addr(); - let end_addr = block.get_end_addr().unwrap(); + let end_addr = block.get_end_addr(); let code_size = block.code_size(); // Write some info about the current block @@ -110,7 +114,7 @@ pub fn disasm_iseq_insn_range(iseq: IseqPtr, start_idx: u16, end_idx: u16) -> St // If this is not the last block if block_idx < block_list.len() - 1 { // Compute the size of the gap between this block and the next - let next_block = block_list[block_idx + 1].borrow(); + let next_block = block_list[block_idx + 1]; let next_start_addr = next_block.get_start_addr(); let gap_size = next_start_addr.into_usize() - end_addr.into_usize(); @@ -318,7 +322,9 @@ fn insns_compiled(iseq: IseqPtr) -> Vec<(String, u16)> { // For each block associated with this iseq for blockref in &block_list { - let block = blockref.borrow(); + // SAFETY: Called as part of a Ruby method, which ensures the graph is + // well connected for the given iseq. + let block = unsafe { blockref.as_ref() }; let start_idx = block.get_blockid().idx; let end_idx = block.get_end_idx(); assert!(u32::from(end_idx) <= unsafe { get_iseq_encoded_size(iseq) }); diff --git a/yjit/src/invariants.rs b/yjit/src/invariants.rs index dbeafe1969..0a969905dc 100644 --- a/yjit/src/invariants.rs +++ b/yjit/src/invariants.rs @@ -16,8 +16,8 @@ use std::mem; // Invariants to track: // assume_bop_not_redefined(jit, INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) // assume_method_lookup_stable(comptime_recv_klass, cme, jit); -// assume_single_ractor_mode(jit) -// assume_stable_global_constant_state(jit); +// assume_single_ractor_mode() +// track_stable_constant_names_assumption() /// Used to track all of the various block references that contain assumptions /// about the state of the virtual machine. @@ -78,9 +78,9 @@ impl Invariants { } } -/// A public function that can be called from within the code generation -/// functions to ensure that the block being generated is invalidated when the -/// basic operator is redefined. +/// Mark the pending block as assuming that certain basic operators (e.g. Integer#==) +/// have not been redefined. +#[must_use] pub fn assume_bop_not_redefined( jit: &mut JITState, ocb: &mut OutlinedCb, @@ -89,18 +89,7 @@ pub fn assume_bop_not_redefined( ) -> bool { if unsafe { BASIC_OP_UNREDEFINED_P(bop, klass) } { jit_ensure_block_entry_exit(jit, ocb); - - let invariants = Invariants::get_instance(); - invariants - .basic_operator_blocks - .entry((klass, bop)) - .or_default() - .insert(jit.get_block()); - invariants - .block_basic_operators - .entry(jit.get_block()) - .or_default() - .insert((klass, bop)); + jit.bop_assumptions.push((klass, bop)); return true; } else { @@ -108,28 +97,33 @@ pub fn assume_bop_not_redefined( } } -// Remember that a block assumes that -// `rb_callable_method_entry(receiver_klass, cme->called_id) == cme` and that -// `cme` is valid. -// When either of these assumptions becomes invalid, rb_yjit_method_lookup_change() or -// rb_yjit_cme_invalidate() invalidates the block. -// -// @raise NoMemoryError -pub fn assume_method_lookup_stable( - jit: &mut JITState, - ocb: &mut OutlinedCb, +/// Track that a block is only valid when a certain basic operator has not been redefined +/// since the block's inception. +pub fn track_bop_assumption(uninit_block: BlockRef, bop: (RedefinitionFlag, ruby_basic_operators)) { + let invariants = Invariants::get_instance(); + invariants + .basic_operator_blocks + .entry(bop) + .or_default() + .insert(uninit_block); + invariants + .block_basic_operators + .entry(uninit_block) + .or_default() + .insert(bop); +} + +/// Track that a block will assume that `cme` is valid (false == METHOD_ENTRY_INVALIDATED(cme)). +/// [rb_yjit_cme_invalidate] invalidates the block when `cme` is invalidated. +pub fn track_method_lookup_stability_assumption( + uninit_block: BlockRef, callee_cme: *const rb_callable_method_entry_t, ) { - jit_ensure_block_entry_exit(jit, ocb); - - let block = jit.get_block(); - jit.push_cme_dependency(callee_cme); - Invariants::get_instance() .cme_validity .entry(callee_cme) .or_default() - .insert(block); + .insert(uninit_block); } // Checks rb_method_basic_definition_p and registers the current block for invalidation if method @@ -141,10 +135,10 @@ pub fn assume_method_basic_definition( ocb: &mut OutlinedCb, klass: VALUE, mid: ID - ) -> bool { +) -> bool { if unsafe { rb_method_basic_definition_p(klass, mid) } != 0 { let cme = unsafe { rb_callable_method_entry(klass, mid) }; - assume_method_lookup_stable(jit, ocb, cme); + jit.assume_method_lookup_stable(ocb, cme); true } else { false @@ -158,22 +152,24 @@ pub fn assume_single_ractor_mode(jit: &mut JITState, ocb: &mut OutlinedCb) -> bo false } else { jit_ensure_block_entry_exit(jit, ocb); - Invariants::get_instance() - .single_ractor - .insert(jit.get_block()); + jit.block_assumes_single_ractor = true; + true } } -/// Walk through the ISEQ to go from the current opt_getinlinecache to the -/// subsequent opt_setinlinecache and find all of the name components that are -/// associated with this constant (which correspond to the getconstant -/// arguments). -pub fn assume_stable_constant_names(jit: &mut JITState, ocb: &mut OutlinedCb, idlist: *const ID) { - /// Tracks that a block is assuming that the name component of a constant - /// has not changed since the last call to this function. +/// Track that the block will assume single ractor mode. +pub fn track_single_ractor_assumption(uninit_block: BlockRef) { + Invariants::get_instance() + .single_ractor + .insert(uninit_block); +} + +/// Track that a block will assume that the name components of a constant path expression +/// has not changed since the block's full initialization. +pub fn track_stable_constant_names_assumption(uninit_block: BlockRef, idlist: *const ID) { fn assume_stable_constant_name( - jit: &mut JITState, + uninit_block: BlockRef, id: ID, ) { if id == idNULL as u64 { @@ -186,10 +182,10 @@ pub fn assume_stable_constant_names(jit: &mut JITState, ocb: &mut OutlinedCb, id .constant_state_blocks .entry(id) .or_default() - .insert(jit.get_block()); + .insert(uninit_block); invariants .block_constant_states - .entry(jit.get_block()) + .entry(uninit_block) .or_default() .insert(id); } @@ -198,12 +194,9 @@ pub fn assume_stable_constant_names(jit: &mut JITState, ocb: &mut OutlinedCb, id for i in 0.. { match unsafe { *idlist.offset(i) } { 0 => break, // End of NULL terminated list - id => assume_stable_constant_name(jit, id), + id => assume_stable_constant_name(uninit_block, id), } } - - jit_ensure_block_entry_exit(jit, ocb); - } /// Called when a basic operator is redefined. Note that all the blocks assuming @@ -344,19 +337,22 @@ pub extern "C" fn rb_yjit_root_mark() { /// Remove all invariant assumptions made by the block by removing the block as /// as a key in all of the relevant tables. -pub fn block_assumptions_free(blockref: &BlockRef) { +/// For safety, the block has to be initialized and the vm lock must be held. +/// However, outgoing/incoming references to the block does _not_ need to be valid. +pub fn block_assumptions_free(blockref: BlockRef) { let invariants = Invariants::get_instance(); { - let block = blockref.borrow(); + // SAFETY: caller ensures that this reference is valid + let block = unsafe { blockref.as_ref() }; // For each method lookup dependency for dep in block.iter_cme_deps() { // Remove tracking for cme validity - if let Some(blockset) = invariants.cme_validity.get_mut(dep) { - blockset.remove(blockref); + if let Some(blockset) = invariants.cme_validity.get_mut(&dep) { + blockset.remove(&blockref); if blockset.is_empty() { - invariants.cme_validity.remove(dep); + invariants.cme_validity.remove(&dep); } } } @@ -506,17 +502,18 @@ pub extern "C" fn rb_yjit_tracing_invalidate_all() { if on_stack_iseqs.contains(&iseq) { // This ISEQ is running, so we can't free blocks immediately for block in blocks { - delayed_deallocation(&block); + delayed_deallocation(block); } payload.dead_blocks.shrink_to_fit(); } else { // Safe to free dead blocks since the ISEQ isn't running + // Since we're freeing _all_ blocks, we don't need to keep the graph well formed for block in blocks { - free_block(&block); + unsafe { free_block(block, false) }; } mem::take(&mut payload.dead_blocks) - .iter() - .for_each(free_block); + .into_iter() + .for_each(|block| unsafe { free_block(block, false) }); } } diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs index b213af8503..785dc9b0f9 100644 --- a/yjit/src/stats.rs +++ b/yjit/src/stats.rs @@ -534,11 +534,11 @@ fn get_live_context_count() -> usize { for_each_iseq_payload(|iseq_payload| { for blocks in iseq_payload.version_map.iter() { for block in blocks.iter() { - count += block.borrow().get_ctx_count(); + count += unsafe { block.as_ref() }.get_ctx_count(); } } for block in iseq_payload.dead_blocks.iter() { - count += block.borrow().get_ctx_count(); + count += unsafe { block.as_ref() }.get_ctx_count(); } }); count |