-stats.rb

-# frozen_string_literal: true

-module RubyVM::RJIT

- # 8-bit memory access

- class BytePtr < Data.define(:reg, :disp); end

-

- # 32-bit memory access

- class DwordPtr < Data.define(:reg, :disp); end

-

- # 64-bit memory access

- QwordPtr = Array

-

- # SystemV x64 calling convention

- C_ARGS = [:rdi, :rsi, :rdx, :rcx, :r8, :r9]

- C_RET = :rax

-

- # https://cdrdv2.intel.com/v1/dl/getContent/671110

- # Mostly an x86_64 assembler, but this also has some stuff that is useful for any architecture.

- class Assembler

- # rel8 jumps are made with labels

- class Label < Data.define(:id, :name); end

-

- # rel32 is inserted as [Rel32, Rel32Pad..] and converted on #resolve_rel32

- class Rel32 < Data.define(:addr); end

- Rel32Pad = Object.new

-

- # A set of ModR/M values encoded on #insn

- class ModRM < Data.define(:mod, :reg, :rm); end

- Mod00 = 0b00 # Mod 00: [reg]

- Mod01 = 0b01 # Mod 01: [reg]+disp8

- Mod10 = 0b10 # Mod 10: [reg]+disp32

- Mod11 = 0b11 # Mod 11: reg

-

- # REX = 0100WR0B

- REX_B = 0b01000001

- REX_R = 0b01000100

- REX_W = 0b01001000

-

- # Operand matchers

- R32 = -> (op) { op.is_a?(Symbol) && r32?(op) }

- R64 = -> (op) { op.is_a?(Symbol) && r64?(op) }

- IMM8 = -> (op) { op.is_a?(Integer) && imm8?(op) }

- IMM32 = -> (op) { op.is_a?(Integer) && imm32?(op) }

- IMM64 = -> (op) { op.is_a?(Integer) && imm64?(op) }

-

- def initialize

- @bytes = []

- @labels = {}

- @label_id = 0

- @comments = Hash.new { |h, k| h[k] = [] }

- @blocks = Hash.new { |h, k| h[k] = [] }

- @stub_starts = Hash.new { |h, k| h[k] = [] }

- @stub_ends = Hash.new { |h, k| h[k] = [] }

- @pos_markers = Hash.new { |h, k| h[k] = [] }

- end

-

- def assemble(addr)

- set_code_addrs(addr)

- resolve_rel32(addr)

- resolve_labels

-

- write_bytes(addr)

-

- @pos_markers.each do |write_pos, markers|

- markers.each { |marker| marker.call(addr + write_pos) }

- end

- @bytes.size

- ensure

- @bytes.clear

- end

-

- def size

- @bytes.size

- end

-

- #

- # Instructions

- #

-

- def add(dst, src)

- case [dst, src]

- # ADD r/m64, imm8 (Mod 00: [reg])

- in [QwordPtr[R64 => dst_reg], IMM8 => src_imm]

- # REX.W + 83 /0 ib

- # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0x83,

- mod_rm: ModRM[mod: Mod00, reg: 0, rm: dst_reg],

- imm: imm8(src_imm),

- )

- # ADD r/m64, imm8 (Mod 11: reg)

- in [R64 => dst_reg, IMM8 => src_imm]

- # REX.W + 83 /0 ib

- # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0x83,

- mod_rm: ModRM[mod: Mod11, reg: 0, rm: dst_reg],

- imm: imm8(src_imm),

- )

- # ADD r/m64 imm32 (Mod 11: reg)

- in [R64 => dst_reg, IMM32 => src_imm]

- # REX.W + 81 /0 id

- # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0x81,

- mod_rm: ModRM[mod: Mod11, reg: 0, rm: dst_reg],

- imm: imm32(src_imm),

- )

- # ADD r/m64, r64 (Mod 11: reg)

- in [R64 => dst_reg, R64 => src_reg]

- # REX.W + 01 /r

- # MR: Operand 1: ModRM:r/m (r, w), Operand 2: ModRM:reg (r)

- insn(

- prefix: REX_W,

- opcode: 0x01,

- mod_rm: ModRM[mod: Mod11, reg: src_reg, rm: dst_reg],

- )

- end

- end

-

- def and(dst, src)

- case [dst, src]

- # AND r/m64, imm8 (Mod 11: reg)

- in [R64 => dst_reg, IMM8 => src_imm]

- # REX.W + 83 /4 ib

- # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0x83,

- mod_rm: ModRM[mod: Mod11, reg: 4, rm: dst_reg],

- imm: imm8(src_imm),

- )

- # AND r/m64, imm32 (Mod 11: reg)

- in [R64 => dst_reg, IMM32 => src_imm]

- # REX.W + 81 /4 id

- # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0x81,

- mod_rm: ModRM[mod: Mod11, reg: 4, rm: dst_reg],

- imm: imm32(src_imm),

- )

- # AND r64, r/m64 (Mod 01: [reg]+disp8)

- in [R64 => dst_reg, QwordPtr[R64 => src_reg, IMM8 => src_disp]]

- # REX.W + 23 /r

- # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: 0x23,

- mod_rm: ModRM[mod: Mod01, reg: dst_reg, rm: src_reg],

- disp: imm8(src_disp),

- )

- # AND r64, r/m64 (Mod 10: [reg]+disp32)

- in [R64 => dst_reg, QwordPtr[R64 => src_reg, IMM32 => src_disp]]

- # REX.W + 23 /r

- # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: 0x23,

- mod_rm: ModRM[mod: Mod10, reg: dst_reg, rm: src_reg],

- disp: imm32(src_disp),

- )

- end

- end

-

- def call(dst)

- case dst

- # CALL rel32

- in Integer => dst_addr

- # E8 cd

- # D: Operand 1: Offset

- insn(opcode: 0xe8, imm: rel32(dst_addr))

- # CALL r/m64 (Mod 11: reg)

- in R64 => dst_reg

- # FF /2

- # M: Operand 1: ModRM:r/m (r)

- insn(

- opcode: 0xff,

- mod_rm: ModRM[mod: Mod11, reg: 2, rm: dst_reg],

- )

- end

- end

-

- def cmove(dst, src)

- case [dst, src]

- # CMOVE r64, r/m64 (Mod 11: reg)

- in [R64 => dst_reg, R64 => src_reg]

- # REX.W + 0F 44 /r

- # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: [0x0f, 0x44],

- mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg],

- )

- end

- end

-

- def cmovg(dst, src)

- case [dst, src]

- # CMOVG r64, r/m64 (Mod 11: reg)

- in [R64 => dst_reg, R64 => src_reg]

- # REX.W + 0F 4F /r

- # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: [0x0f, 0x4f],

- mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg],

- )

- end

- end

-

- def cmovge(dst, src)

- case [dst, src]

- # CMOVGE r64, r/m64 (Mod 11: reg)

- in [R64 => dst_reg, R64 => src_reg]

- # REX.W + 0F 4D /r

- # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: [0x0f, 0x4d],

- mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg],

- )

- end

- end

-

- def cmovl(dst, src)

- case [dst, src]

- # CMOVL r64, r/m64 (Mod 11: reg)

- in [R64 => dst_reg, R64 => src_reg]

- # REX.W + 0F 4C /r

- # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: [0x0f, 0x4c],

- mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg],

- )

- end

- end

-

- def cmovle(dst, src)

- case [dst, src]

- # CMOVLE r64, r/m64 (Mod 11: reg)

- in [R64 => dst_reg, R64 => src_reg]

- # REX.W + 0F 4E /r

- # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: [0x0f, 0x4e],

- mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg],

- )

- end

- end

-

- def cmovne(dst, src)

- case [dst, src]

- # CMOVNE r64, r/m64 (Mod 11: reg)

- in [R64 => dst_reg, R64 => src_reg]

- # REX.W + 0F 45 /r

- # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: [0x0f, 0x45],

- mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg],

- )

- end

- end

-

- def cmovnz(dst, src)

- case [dst, src]

- # CMOVNZ r64, r/m64 (Mod 11: reg)

- in [R64 => dst_reg, R64 => src_reg]

- # REX.W + 0F 45 /r

- # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: [0x0f, 0x45],

- mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg],

- )

- end

- end

-

- def cmovz(dst, src)

- case [dst, src]

- # CMOVZ r64, r/m64 (Mod 11: reg)

- in [R64 => dst_reg, R64 => src_reg]

- # REX.W + 0F 44 /r

- # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: [0x0f, 0x44],

- mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg],

- )

- # CMOVZ r64, r/m64 (Mod 01: [reg]+disp8)

- in [R64 => dst_reg, QwordPtr[R64 => src_reg, IMM8 => src_disp]]

- # REX.W + 0F 44 /r

- # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: [0x0f, 0x44],

- mod_rm: ModRM[mod: Mod01, reg: dst_reg, rm: src_reg],

- disp: imm8(src_disp),

- )

- end

- end

-

- def cmp(left, right)

- case [left, right]

- # CMP r/m8, imm8 (Mod 01: [reg]+disp8)

- in [BytePtr[R64 => left_reg, IMM8 => left_disp], IMM8 => right_imm]

- # 80 /7 ib

- # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32

- insn(

- opcode: 0x80,

- mod_rm: ModRM[mod: Mod01, reg: 7, rm: left_reg],

- disp: left_disp,

- imm: imm8(right_imm),

- )

- # CMP r/m32, imm32 (Mod 01: [reg]+disp8)

- in [DwordPtr[R64 => left_reg, IMM8 => left_disp], IMM32 => right_imm]

- # 81 /7 id

- # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32

- insn(

- opcode: 0x81,

- mod_rm: ModRM[mod: Mod01, reg: 7, rm: left_reg],

- disp: left_disp,

- imm: imm32(right_imm),

- )

- # CMP r/m64, imm8 (Mod 01: [reg]+disp8)

- in [QwordPtr[R64 => left_reg, IMM8 => left_disp], IMM8 => right_imm]

- # REX.W + 83 /7 ib

- # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0x83,

- mod_rm: ModRM[mod: Mod01, reg: 7, rm: left_reg],

- disp: left_disp,

- imm: imm8(right_imm),

- )

- # CMP r/m64, imm32 (Mod 01: [reg]+disp8)

- in [QwordPtr[R64 => left_reg, IMM8 => left_disp], IMM32 => right_imm]

- # REX.W + 81 /7 id

- # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0x81,

- mod_rm: ModRM[mod: Mod01, reg: 7, rm: left_reg],

- disp: left_disp,

- imm: imm32(right_imm),

- )

- # CMP r/m64, imm8 (Mod 10: [reg]+disp32)

- in [QwordPtr[R64 => left_reg, IMM32 => left_disp], IMM8 => right_imm]

- # REX.W + 83 /7 ib

- # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0x83,

- mod_rm: ModRM[mod: Mod10, reg: 7, rm: left_reg],

- disp: imm32(left_disp),

- imm: imm8(right_imm),

- )

- # CMP r/m64, imm8 (Mod 11: reg)

- in [R64 => left_reg, IMM8 => right_imm]

- # REX.W + 83 /7 ib

- # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0x83,

- mod_rm: ModRM[mod: Mod11, reg: 7, rm: left_reg],

- imm: imm8(right_imm),

- )

- # CMP r/m64, imm32 (Mod 11: reg)

- in [R64 => left_reg, IMM32 => right_imm]

- # REX.W + 81 /7 id

- # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0x81,

- mod_rm: ModRM[mod: Mod11, reg: 7, rm: left_reg],

- imm: imm32(right_imm),

- )

- # CMP r/m64, r64 (Mod 01: [reg]+disp8)

- in [QwordPtr[R64 => left_reg, IMM8 => left_disp], R64 => right_reg]

- # REX.W + 39 /r

- # MR: Operand 1: ModRM:r/m (r), Operand 2: ModRM:reg (r)

- insn(

- prefix: REX_W,

- opcode: 0x39,

- mod_rm: ModRM[mod: Mod01, reg: right_reg, rm: left_reg],

- disp: left_disp,

- )

- # CMP r/m64, r64 (Mod 10: [reg]+disp32)

- in [QwordPtr[R64 => left_reg, IMM32 => left_disp], R64 => right_reg]

- # REX.W + 39 /r

- # MR: Operand 1: ModRM:r/m (r), Operand 2: ModRM:reg (r)

- insn(

- prefix: REX_W,

- opcode: 0x39,

- mod_rm: ModRM[mod: Mod10, reg: right_reg, rm: left_reg],

- disp: imm32(left_disp),

- )

- # CMP r/m64, r64 (Mod 11: reg)

- in [R64 => left_reg, R64 => right_reg]

- # REX.W + 39 /r

- # MR: Operand 1: ModRM:r/m (r), Operand 2: ModRM:reg (r)

- insn(

- prefix: REX_W,

- opcode: 0x39,

- mod_rm: ModRM[mod: Mod11, reg: right_reg, rm: left_reg],

- )

- end

- end

-

- def jbe(dst)

- case dst

- # JBE rel8

- in Label => dst_label

- # 76 cb

- insn(opcode: 0x76, imm: dst_label)

- # JBE rel32

- in Integer => dst_addr

- # 0F 86 cd

- insn(opcode: [0x0f, 0x86], imm: rel32(dst_addr))

- end

- end

-

- def je(dst)

- case dst

- # JE rel8

- in Label => dst_label

- # 74 cb

- insn(opcode: 0x74, imm: dst_label)

- # JE rel32

- in Integer => dst_addr

- # 0F 84 cd

- insn(opcode: [0x0f, 0x84], imm: rel32(dst_addr))

- end

- end

-

- def jl(dst)

- case dst

- # JL rel32

- in Integer => dst_addr

- # 0F 8C cd

- insn(opcode: [0x0f, 0x8c], imm: rel32(dst_addr))

- end

- end

-

- def jmp(dst)

- case dst

- # JZ rel8

- in Label => dst_label

- # EB cb

- insn(opcode: 0xeb, imm: dst_label)

- # JMP rel32

- in Integer => dst_addr

- # E9 cd

- insn(opcode: 0xe9, imm: rel32(dst_addr))

- # JMP r/m64 (Mod 01: [reg]+disp8)

- in QwordPtr[R64 => dst_reg, IMM8 => dst_disp]

- # FF /4

- insn(opcode: 0xff, mod_rm: ModRM[mod: Mod01, reg: 4, rm: dst_reg], disp: dst_disp)

- # JMP r/m64 (Mod 11: reg)

- in R64 => dst_reg

- # FF /4

- insn(opcode: 0xff, mod_rm: ModRM[mod: Mod11, reg: 4, rm: dst_reg])

- end

- end

-

- def jne(dst)

- case dst

- # JNE rel8

- in Label => dst_label

- # 75 cb

- insn(opcode: 0x75, imm: dst_label)

- # JNE rel32

- in Integer => dst_addr

- # 0F 85 cd

- insn(opcode: [0x0f, 0x85], imm: rel32(dst_addr))

- end

- end

-

- def jnz(dst)

- case dst

- # JE rel8

- in Label => dst_label

- # 75 cb

- insn(opcode: 0x75, imm: dst_label)

- # JNZ rel32

- in Integer => dst_addr

- # 0F 85 cd

- insn(opcode: [0x0f, 0x85], imm: rel32(dst_addr))

- end

- end

-

- def jo(dst)

- case dst

- # JO rel32

- in Integer => dst_addr

- # 0F 80 cd

- insn(opcode: [0x0f, 0x80], imm: rel32(dst_addr))

- end

- end

-

- def jz(dst)

- case dst

- # JZ rel8

- in Label => dst_label

- # 74 cb

- insn(opcode: 0x74, imm: dst_label)

- # JZ rel32

- in Integer => dst_addr

- # 0F 84 cd

- insn(opcode: [0x0f, 0x84], imm: rel32(dst_addr))

- end

- end

-

- def lea(dst, src)

- case [dst, src]

- # LEA r64,m (Mod 01: [reg]+disp8)

- in [R64 => dst_reg, QwordPtr[R64 => src_reg, IMM8 => src_disp]]

- # REX.W + 8D /r

- # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: 0x8d,

- mod_rm: ModRM[mod: Mod01, reg: dst_reg, rm: src_reg],

- disp: imm8(src_disp),

- )

- # LEA r64,m (Mod 10: [reg]+disp32)

- in [R64 => dst_reg, QwordPtr[R64 => src_reg, IMM32 => src_disp]]

- # REX.W + 8D /r

- # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: 0x8d,

- mod_rm: ModRM[mod: Mod10, reg: dst_reg, rm: src_reg],

- disp: imm32(src_disp),

- )

- end

- end

-

- def mov(dst, src)

- case dst

- in R32 => dst_reg

- case src

- # MOV r32 r/m32 (Mod 01: [reg]+disp8)

- in DwordPtr[R64 => src_reg, IMM8 => src_disp]

- # 8B /r

- # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r)

- insn(

- opcode: 0x8b,

- mod_rm: ModRM[mod: Mod01, reg: dst_reg, rm: src_reg],

- disp: src_disp,

- )

- # MOV r32, imm32 (Mod 11: reg)

- in IMM32 => src_imm

- # B8+ rd id

- # OI: Operand 1: opcode + rd (w), Operand 2: imm8/16/32/64

- insn(

- opcode: 0xb8,

- rd: dst_reg,

- imm: imm32(src_imm),

- )

- end

- in R64 => dst_reg

- case src

- # MOV r64, r/m64 (Mod 00: [reg])

- in QwordPtr[R64 => src_reg]

- # REX.W + 8B /r

- # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: 0x8b,

- mod_rm: ModRM[mod: Mod00, reg: dst_reg, rm: src_reg],

- )

- # MOV r64, r/m64 (Mod 01: [reg]+disp8)

- in QwordPtr[R64 => src_reg, IMM8 => src_disp]

- # REX.W + 8B /r

- # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: 0x8b,

- mod_rm: ModRM[mod: Mod01, reg: dst_reg, rm: src_reg],

- disp: src_disp,

- )

- # MOV r64, r/m64 (Mod 10: [reg]+disp32)

- in QwordPtr[R64 => src_reg, IMM32 => src_disp]

- # REX.W + 8B /r

- # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: 0x8b,

- mod_rm: ModRM[mod: Mod10, reg: dst_reg, rm: src_reg],

- disp: imm32(src_disp),

- )

- # MOV r64, r/m64 (Mod 11: reg)

- in R64 => src_reg

- # REX.W + 8B /r

- # RM: Operand 1: ModRM:reg (w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: 0x8b,

- mod_rm: ModRM[mod: Mod11, reg: dst_reg, rm: src_reg],

- )

- # MOV r/m64, imm32 (Mod 11: reg)

- in IMM32 => src_imm

- # REX.W + C7 /0 id

- # MI: Operand 1: ModRM:r/m (w), Operand 2: imm8/16/32/64

- insn(

- prefix: REX_W,

- opcode: 0xc7,

- mod_rm: ModRM[mod: Mod11, reg: 0, rm: dst_reg],

- imm: imm32(src_imm),

- )

- # MOV r64, imm64

- in IMM64 => src_imm

- # REX.W + B8+ rd io

- # OI: Operand 1: opcode + rd (w), Operand 2: imm8/16/32/64

- insn(

- prefix: REX_W,

- opcode: 0xb8,

- rd: dst_reg,

- imm: imm64(src_imm),

- )

- end

- in DwordPtr[R64 => dst_reg, IMM8 => dst_disp]

- case src

- # MOV r/m32, imm32 (Mod 01: [reg]+disp8)

- in IMM32 => src_imm

- # C7 /0 id

- # MI: Operand 1: ModRM:r/m (w), Operand 2: imm8/16/32/64

- insn(

- opcode: 0xc7,

- mod_rm: ModRM[mod: Mod01, reg: 0, rm: dst_reg],

- disp: dst_disp,

- imm: imm32(src_imm),

- )

- end

- in QwordPtr[R64 => dst_reg]

- case src

- # MOV r/m64, imm32 (Mod 00: [reg])

- in IMM32 => src_imm

- # REX.W + C7 /0 id

- # MI: Operand 1: ModRM:r/m (w), Operand 2: imm8/16/32/64

- insn(

- prefix: REX_W,

- opcode: 0xc7,

- mod_rm: ModRM[mod: Mod00, reg: 0, rm: dst_reg],

- imm: imm32(src_imm),

- )

- # MOV r/m64, r64 (Mod 00: [reg])

- in R64 => src_reg

- # REX.W + 89 /r

- # MR: Operand 1: ModRM:r/m (w), Operand 2: ModRM:reg (r)

- insn(

- prefix: REX_W,

- opcode: 0x89,

- mod_rm: ModRM[mod: Mod00, reg: src_reg, rm: dst_reg],

- )

- end

- in QwordPtr[R64 => dst_reg, IMM8 => dst_disp]

- # Optimize encoding when disp is 0

- return mov([dst_reg], src) if dst_disp == 0

-

- case src

- # MOV r/m64, imm32 (Mod 01: [reg]+disp8)

- in IMM32 => src_imm

- # REX.W + C7 /0 id

- # MI: Operand 1: ModRM:r/m (w), Operand 2: imm8/16/32/64

- insn(

- prefix: REX_W,

- opcode: 0xc7,

- mod_rm: ModRM[mod: Mod01, reg: 0, rm: dst_reg],

- disp: dst_disp,

- imm: imm32(src_imm),

- )

- # MOV r/m64, r64 (Mod 01: [reg]+disp8)

- in R64 => src_reg

- # REX.W + 89 /r

- # MR: Operand 1: ModRM:r/m (w), Operand 2: ModRM:reg (r)

- insn(

- prefix: REX_W,

- opcode: 0x89,

- mod_rm: ModRM[mod: Mod01, reg: src_reg, rm: dst_reg],

- disp: dst_disp,

- )

- end

- in QwordPtr[R64 => dst_reg, IMM32 => dst_disp]

- case src

- # MOV r/m64, imm32 (Mod 10: [reg]+disp32)

- in IMM32 => src_imm

- # REX.W + C7 /0 id

- # MI: Operand 1: ModRM:r/m (w), Operand 2: imm8/16/32/64

- insn(

- prefix: REX_W,

- opcode: 0xc7,

- mod_rm: ModRM[mod: Mod10, reg: 0, rm: dst_reg],

- disp: imm32(dst_disp),

- imm: imm32(src_imm),

- )

- # MOV r/m64, r64 (Mod 10: [reg]+disp32)

- in R64 => src_reg

- # REX.W + 89 /r

- # MR: Operand 1: ModRM:r/m (w), Operand 2: ModRM:reg (r)

- insn(

- prefix: REX_W,

- opcode: 0x89,

- mod_rm: ModRM[mod: Mod10, reg: src_reg, rm: dst_reg],

- disp: imm32(dst_disp),

- )

- end

- end

- end

-

- def or(dst, src)

- case [dst, src]

- # OR r/m64, imm8 (Mod 11: reg)

- in [R64 => dst_reg, IMM8 => src_imm]

- # REX.W + 83 /1 ib

- # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0x83,

- mod_rm: ModRM[mod: Mod11, reg: 1, rm: dst_reg],

- imm: imm8(src_imm),

- )

- # OR r/m64, imm32 (Mod 11: reg)

- in [R64 => dst_reg, IMM32 => src_imm]

- # REX.W + 81 /1 id

- # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0x81,

- mod_rm: ModRM[mod: Mod11, reg: 1, rm: dst_reg],

- imm: imm32(src_imm),

- )

- # OR r64, r/m64 (Mod 01: [reg]+disp8)

- in [R64 => dst_reg, QwordPtr[R64 => src_reg, IMM8 => src_disp]]

- # REX.W + 0B /r

- # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: 0x0b,

- mod_rm: ModRM[mod: Mod01, reg: dst_reg, rm: src_reg],

- disp: imm8(src_disp),

- )

- # OR r64, r/m64 (Mod 10: [reg]+disp32)

- in [R64 => dst_reg, QwordPtr[R64 => src_reg, IMM32 => src_disp]]

- # REX.W + 0B /r

- # RM: Operand 1: ModRM:reg (r, w), Operand 2: ModRM:r/m (r)

- insn(

- prefix: REX_W,

- opcode: 0x0b,

- mod_rm: ModRM[mod: Mod10, reg: dst_reg, rm: src_reg],

- disp: imm32(src_disp),

- )

- end

- end

-

- def push(src)

- case src

- # PUSH r64

- in R64 => src_reg

- # 50+rd

- # O: Operand 1: opcode + rd (r)

- insn(opcode: 0x50, rd: src_reg)

- end

- end

-

- def pop(dst)

- case dst

- # POP r64

- in R64 => dst_reg

- # 58+ rd

- # O: Operand 1: opcode + rd (r)

- insn(opcode: 0x58, rd: dst_reg)

- end

- end

-

- def ret

- # RET

- # Near return: A return to a procedure within the current code segment

- insn(opcode: 0xc3)

- end

-

- def sar(dst, src)

- case [dst, src]

- in [R64 => dst_reg, IMM8 => src_imm]

- # REX.W + C1 /7 ib

- # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8

- insn(

- prefix: REX_W,

- opcode: 0xc1,

- mod_rm: ModRM[mod: Mod11, reg: 7, rm: dst_reg],

- imm: imm8(src_imm),

- )

- end

- end

-

- def sub(dst, src)

- case [dst, src]

- # SUB r/m64, imm8 (Mod 11: reg)

- in [R64 => dst_reg, IMM8 => src_imm]

- # REX.W + 83 /5 ib

- # MI: Operand 1: ModRM:r/m (r, w), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0x83,

- mod_rm: ModRM[mod: Mod11, reg: 5, rm: dst_reg],

- imm: imm8(src_imm),

- )

- # SUB r/m64, r64 (Mod 11: reg)

- in [R64 => dst_reg, R64 => src_reg]

- # REX.W + 29 /r

- # MR: Operand 1: ModRM:r/m (r, w), Operand 2: ModRM:reg (r)

- insn(

- prefix: REX_W,

- opcode: 0x29,

- mod_rm: ModRM[mod: Mod11, reg: src_reg, rm: dst_reg],

- )

- end

- end

-

- def test(left, right)

- case [left, right]

- # TEST r/m8*, imm8 (Mod 01: [reg]+disp8)

- in [BytePtr[R64 => left_reg, IMM8 => left_disp], IMM8 => right_imm]

- # REX + F6 /0 ib

- # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32

- insn(

- opcode: 0xf6,

- mod_rm: ModRM[mod: Mod01, reg: 0, rm: left_reg],

- disp: left_disp,

- imm: imm8(right_imm),

- )

- # TEST r/m64, imm32 (Mod 01: [reg]+disp8)

- in [QwordPtr[R64 => left_reg, IMM8 => left_disp], IMM32 => right_imm]

- # REX.W + F7 /0 id

- # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0xf7,

- mod_rm: ModRM[mod: Mod01, reg: 0, rm: left_reg],

- disp: left_disp,

- imm: imm32(right_imm),

- )

- # TEST r/m64, imm32 (Mod 10: [reg]+disp32)

- in [QwordPtr[R64 => left_reg, IMM32 => left_disp], IMM32 => right_imm]

- # REX.W + F7 /0 id

- # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0xf7,

- mod_rm: ModRM[mod: Mod10, reg: 0, rm: left_reg],

- disp: imm32(left_disp),

- imm: imm32(right_imm),

- )

- # TEST r/m64, imm32 (Mod 11: reg)

- in [R64 => left_reg, IMM32 => right_imm]

- # REX.W + F7 /0 id

- # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32

- insn(

- prefix: REX_W,

- opcode: 0xf7,

- mod_rm: ModRM[mod: Mod11, reg: 0, rm: left_reg],

- imm: imm32(right_imm),

- )

- # TEST r/m32, r32 (Mod 11: reg)

- in [R32 => left_reg, R32 => right_reg]

- # 85 /r

- # MR: Operand 1: ModRM:r/m (r), Operand 2: ModRM:reg (r)

- insn(

- opcode: 0x85,

- mod_rm: ModRM[mod: Mod11, reg: right_reg, rm: left_reg],

- )

- # TEST r/m64, r64 (Mod 11: reg)

- in [R64 => left_reg, R64 => right_reg]

- # REX.W + 85 /r

- # MR: Operand 1: ModRM:r/m (r), Operand 2: ModRM:reg (r)

- insn(

- prefix: REX_W,

- opcode: 0x85,

- mod_rm: ModRM[mod: Mod11, reg: right_reg, rm: left_reg],

- )

- end

- end

-

- def xor(dst, src)

- case [dst, src]

- # XOR r/m64, r64 (Mod 11: reg)

- in [R64 => dst_reg, R64 => src_reg]

- # REX.W + 31 /r

- # MR: Operand 1: ModRM:r/m (r, w), Operand 2: ModRM:reg (r)

- insn(

- prefix: REX_W,

- opcode: 0x31,

- mod_rm: ModRM[mod: Mod11, reg: src_reg, rm: dst_reg],

- )

- end

- end

-

- #

- # Utilities

- #

-

- attr_reader :comments

-

- def comment(message)

- @comments[@bytes.size] << message

- end

-

- # Mark the starting address of a block

- def block(block)

- @blocks[@bytes.size] << block

- end

-

- # Mark the starting/ending addresses of a stub

- def stub(stub)

- @stub_starts[@bytes.size] << stub

- yield

- ensure

- @stub_ends[@bytes.size] << stub

- end

-

- def pos_marker(&block)

- @pos_markers[@bytes.size] << block

- end

-

- def new_label(name)

- Label.new(id: @label_id += 1, name:)

- end

-

- # @param [RubyVM::RJIT::Assembler::Label] label

- def write_label(label)

- @labels[label] = @bytes.size

- end

-

- def incr_counter(name)

- if C.rjit_opts.stats

- comment("increment counter #{name}")

- mov(:rax, C.rb_rjit_counters[name].to_i)

- add([:rax], 1) # TODO: lock

- end

- end

-

- private

-

- def insn(prefix: 0, opcode:, rd: nil, mod_rm: nil, disp: nil, imm: nil)

- # Determine prefix

- if rd

- prefix |= REX_B if extended_reg?(rd)

- opcode += reg_code(rd)

- end

- if mod_rm

- prefix |= REX_R if mod_rm.reg.is_a?(Symbol) && extended_reg?(mod_rm.reg)

- prefix |= REX_B if mod_rm.rm.is_a?(Symbol) && extended_reg?(mod_rm.rm)

- end

-

- # Encode insn

- if prefix > 0

- @bytes.push(prefix)

- end

- @bytes.push(*Array(opcode))

- if mod_rm

- mod_rm_byte = encode_mod_rm(

- mod: mod_rm.mod,

- reg: mod_rm.reg.is_a?(Symbol) ? reg_code(mod_rm.reg) : mod_rm.reg,

- rm: mod_rm.rm.is_a?(Symbol) ? reg_code(mod_rm.rm) : mod_rm.rm,

- )

- @bytes.push(mod_rm_byte)

- end

- if disp

- @bytes.push(*Array(disp))

- end

- if imm

- @bytes.push(*imm)

- end

- end

-

- def reg_code(reg)

- reg_code_extended(reg).first

- end

-

- # Table 2-2. 32-Bit Addressing Forms with the ModR/M Byte

- #

- # 7 6 5 4 3 2 1 0

- # +--+--+--+--+--+--+--+--+

- # | Mod | Reg/ | R/M |

- # | | Opcode | |

- # +--+--+--+--+--+--+--+--+

- #

- # The r/m field can specify a register as an operand or it can be combined

- # with the mod field to encode an addressing mode.

- #

- # /0: R/M is 0 (not used)

- # /r: R/M is a register

- def encode_mod_rm(mod:, reg: 0, rm: 0)

- if mod > 0b11

- raise ArgumentError, "too large Mod: #{mod}"

- end

- if reg > 0b111

- raise ArgumentError, "too large Reg/Opcode: #{reg}"

- end

- if rm > 0b111

- raise ArgumentError, "too large R/M: #{rm}"

- end

- (mod << 6) + (reg << 3) + rm

- end

-

- # ib: 1 byte

- def imm8(imm)

- unless imm8?(imm)

- raise ArgumentError, "unexpected imm8: #{imm}"

- end

- [imm].pack('c').unpack('c*') # TODO: consider uimm

- end

-

- # id: 4 bytes

- def imm32(imm)

- unless imm32?(imm)

- raise ArgumentError, "unexpected imm32: #{imm}"

- end

- [imm].pack('l').unpack('c*') # TODO: consider uimm

- end

-

- # io: 8 bytes

- def imm64(imm)

- unless imm64?(imm)

- raise ArgumentError, "unexpected imm64: #{imm}"

- end

- imm_bytes(imm, 8)

- end

-

- def imm_bytes(imm, num_bytes)

- bytes = []

- bits = imm

- num_bytes.times do

- bytes << (bits & 0xff)

- bits >>= 8

- end

- if bits != 0

- raise ArgumentError, "unexpected imm with #{num_bytes} bytes: #{imm}"

- end

- bytes

- end

-

- def rel32(addr)

- [Rel32.new(addr), Rel32Pad, Rel32Pad, Rel32Pad]

- end

-

- def set_code_addrs(write_addr)

- (@bytes.size + 1).times do |index|

- @blocks.fetch(index, []).each do |block|

- block.start_addr = write_addr + index

- end

- @stub_starts.fetch(index, []).each do |stub|

- stub.start_addr = write_addr + index

- end

- @stub_ends.fetch(index, []).each do |stub|

- stub.end_addr = write_addr + index

- end

- end

- end

-

- def resolve_rel32(write_addr)

- @bytes.each_with_index do |byte, index|

- if byte.is_a?(Rel32)

- src_addr = write_addr + index + 4 # offset 4 bytes for rel32 itself

- dst_addr = byte.addr

- rel32 = dst_addr - src_addr

- raise "unexpected offset: #{rel32}" unless imm32?(rel32)

- imm32(rel32).each_with_index do |rel_byte, rel_index|

- @bytes[index + rel_index] = rel_byte

- end

- end

- end

- end

-

- def resolve_labels

- @bytes.each_with_index do |byte, index|

- if byte.is_a?(Label)

- src_index = index + 1 # offset 1 byte for rel8 itself

- dst_index = @labels.fetch(byte)

- rel8 = dst_index - src_index

- raise "unexpected offset: #{rel8}" unless imm8?(rel8)

- @bytes[index] = rel8

- end

- end

- end

-

- def write_bytes(addr)

- Fiddle::Pointer.new(addr)[0, @bytes.size] = @bytes.pack('c*')

- end

- end

-

- module OperandMatcher

- def imm8?(imm)

- (-0x80..0x7f).include?(imm)

- end

-

- def imm32?(imm)

- (-0x8000_0000..0x7fff_ffff).include?(imm) # TODO: consider uimm

- end

-

- def imm64?(imm)

- (-0x8000_0000_0000_0000..0xffff_ffff_ffff_ffff).include?(imm)

- end

-

- def r32?(reg)

- if extended_reg?(reg)

- reg.end_with?('d')

- else

- reg.start_with?('e')

- end

- end

-

- def r64?(reg)

- if extended_reg?(reg)

- reg.match?(/\Ar\d+\z/)

- else

- reg.start_with?('r')

- end

- end

-

- def extended_reg?(reg)

- reg_code_extended(reg).last

- end

-

- def reg_code_extended(reg)

- case reg

- # Not extended

- when :al, :ax, :eax, :rax then [0, false]

- when :cl, :cx, :ecx, :rcx then [1, false]

- when :dl, :dx, :edx, :rdx then [2, false]

- when :bl, :bx, :ebx, :rbx then [3, false]

- when :ah, :sp, :esp, :rsp then [4, false]

- when :ch, :bp, :ebp, :rbp then [5, false]

- when :dh, :si, :esi, :rsi then [6, false]

- when :bh, :di, :edi, :rdi then [7, false]

- # Extended

- when :r8b, :r8w, :r8d, :r8 then [0, true]

- when :r9b, :r9w, :r9d, :r9 then [1, true]

- when :r10b, :r10w, :r10d, :r10 then [2, true]

- when :r11b, :r11w, :r11d, :r11 then [3, true]

- when :r12b, :r12w, :r12d, :r12 then [4, true]

- when :r13b, :r13w, :r13d, :r13 then [5, true]

- when :r14b, :r14w, :r14d, :r14 then [6, true]

- when :r15b, :r15w, :r15d, :r15 then [7, true]

- else raise ArgumentError, "unexpected reg: #{reg.inspect}"

- end

- end

- end

-

- class Assembler

- include OperandMatcher

- extend OperandMatcher

- end

-end

-class RubyVM::RJIT::Block < Struct.new(

- :iseq, # @param ``

- :pc, # @param [Integer] Starting PC

- :ctx, # @param [RubyVM::RJIT::Context] **Starting** Context (TODO: freeze?)

- :start_addr, # @param [Integer] Starting address of this block's JIT code

- :entry_exit, # @param [Integer] Address of entry exit (optional)

- :incoming, # @param [Array<RubyVM::RJIT::BranchStub>] Incoming branches

- :invalidated, # @param [TrueClass,FalseClass] true if already invalidated

-)

- def initialize(incoming: [], invalidated: false, **) = super

-end

-module RubyVM::RJIT

- # Branch shapes

- Next0 = :Next0 # target0 is a fallthrough

- Next1 = :Next1 # target1 is a fallthrough

- Default = :Default # neither targets is a fallthrough

-

- class BranchStub < Struct.new(

- :iseq, # @param [RubyVM::RJIT::CPointer::Struct_rb_iseq_struct] Branch target ISEQ

- :shape, # @param [Symbol] Next0, Next1, or Default

- :target0, # @param [RubyVM::RJIT::BranchTarget] First branch target

- :target1, # @param [RubyVM::RJIT::BranchTarget,NilClass] Second branch target (optional)

- :compile, # @param [Proc] A callback to (re-)generate this branch stub

- :start_addr, # @param [Integer] Stub source start address to be re-generated

- :end_addr, # @param [Integer] Stub source end address to be re-generated

- )

- end

-

- class BranchTarget < Struct.new(

- :pc,

- :ctx,

- :address,

- )

- end

-end

-module RubyVM::RJIT

- # Every class under this namespace is a pointer. Even if the type is

- # immediate, it shouldn't be dereferenced until `*` is called.

- module CPointer

- # Note: We'd like to avoid alphabetic method names to avoid a conflict

- # with member methods. to_i and to_s are considered an exception.

- class Struct

- # @param name [String]

- # @param sizeof [Integer]

- # @param members [Hash{ Symbol => [RubyVM::RJIT::CType::*, Integer, TrueClass] }]

- def initialize(addr, sizeof, members)

- @addr = addr

- @sizeof = sizeof

- @members = members

- end

-

- # Get a raw address

- def to_i

- @addr

- end

-

- # Serialized address for generated code

- def to_s

- "0x#{@addr.to_s(16)}"

- end

-

- # Pointer diff

- def -(struct)

- raise ArgumentError if self.class != struct.class

- (@addr - struct.to_i) / @sizeof

- end

-

- # Primitive API that does no automatic dereference

- # TODO: remove this?

- # @param member [Symbol]

- def [](member)

- type, offset = @members.fetch(member)

- type.new(@addr + offset / 8)

- end

-

- private

-

- # @param member [Symbol]

- # @param value [Object]

- def []=(member, value)

- type, offset = @members.fetch(member)

- type[@addr + offset / 8] = value

- end

-

- # @param size [Integer]

- # @param members [Hash{ Symbol => [Integer, RubyVM::RJIT::CType::*] }]

- def self.define(size, members)

- Class.new(self) do

- # Return the size of this type

- define_singleton_method(:size) { size }

-

- # Return the offset to a field

- define_singleton_method(:offsetof) do |field, *fields|

- member, offset = members.fetch(field)

- offset /= 8

- unless fields.empty?

- offset += member.offsetof(*fields)

- end

- offset

- end

-

- # Return member names

- define_singleton_method(:members) { members.keys }

-

- define_method(:initialize) do |addr = nil|

- if addr.nil? # TODO: get rid of this feature later

- addr = Fiddle.malloc(size)

- end

- super(addr, size, members)

- end

-

- members.each do |member, (type, offset, to_ruby)|

- # Intelligent API that does automatic dereference

- define_method(member) do

- value = self[member]

- if value.respond_to?(:*)

- value = value.*

- end

- if to_ruby

- value = C.to_ruby(value)

- end

- value

- end

-

- define_method("#{member}=") do |value|

- if to_ruby

- value = C.to_value(value)

- end

- self[member] = value

- end

- end

- end

- end

- end

-

- # Note: We'd like to avoid alphabetic method names to avoid a conflict

- # with member methods. to_i is considered an exception.

- class Union

- # @param _name [String] To be used when it starts defining a union pointer class

- # @param sizeof [Integer]

- # @param members [Hash{ Symbol => RubyVM::RJIT::CType::* }]

- def initialize(addr, sizeof, members)

- @addr = addr

- @sizeof = sizeof

- @members = members

- end

-

- # Get a raw address

- def to_i

- @addr

- end

-

- # Move addr to access this pointer like an array

- def +(index)

- raise ArgumentError unless index.is_a?(Integer)

- self.class.new(@addr + index * @sizeof)

- end

-

- # Pointer diff

- def -(union)

- raise ArgumentError if self.class != union.class

- (@addr - union.instance_variable_get(:@addr)) / @sizeof

- end

-

- # @param sizeof [Integer]

- # @param members [Hash{ Symbol => RubyVM::RJIT::CType::* }]

- def self.define(sizeof, members)

- Class.new(self) do

- # Return the size of this type

- define_singleton_method(:sizeof) { sizeof }

-

- # Part of Struct's offsetof implementation

- define_singleton_method(:offsetof) do |field, *fields|

- member = members.fetch(field)

- offset = 0

- unless fields.empty?

- offset += member.offsetof(*fields)

- end

- offset

- end

-

- define_method(:initialize) do |addr|

- super(addr, sizeof, members)

- end

-

- members.each do |member, type|

- # Intelligent API that does automatic dereference

- define_method(member) do

- value = type.new(@addr)

- if value.respond_to?(:*)

- value = value.*

- end

- value

- end

- end

- end

- end

- end

-

- class Immediate

- # @param addr [Integer]

- # @param size [Integer]

- # @param pack [String]

- def initialize(addr, size, pack)

- @addr = addr

- @size = size

- @pack = pack

- end

-

- # Get a raw address

- def to_i

- @addr

- end

-

- # Move addr to addess this pointer like an array

- def +(index)

- Immediate.new(@addr + index * @size, @size, @pack)

- end

-

- # Dereference

- def *

- self[0]

- end

-

- # Array access

- def [](index)

- return nil if @addr == 0

- Fiddle::Pointer.new(@addr + index * @size)[0, @size].unpack1(@pack)

- end

-

- # Array set

- def []=(index, value)

- Fiddle::Pointer.new(@addr + index * @size)[0, @size] = [value].pack(@pack)

- end

-

- # Serialized address for generated code. Used for embedding things like body->iseq_encoded.

- def to_s

- "0x#{Integer(@addr).to_s(16)}"

- end

-

- # @param fiddle_type [Integer] Fiddle::TYPE_*

- def self.define(fiddle_type)

- size = Fiddle::PackInfo::SIZE_MAP.fetch(fiddle_type)

- pack = Fiddle::PackInfo::PACK_MAP.fetch(fiddle_type)

-

- Class.new(self) do

- define_method(:initialize) do |addr|

- super(addr, size, pack)

- end

-

- define_singleton_method(:size) do

- size

- end

-

- # Type-level []=: Used by struct fields

- define_singleton_method(:[]=) do |addr, value|

- Fiddle::Pointer.new(addr)[0, size] = [value].pack(pack)

- end

- end

- end

- end

-

- # -Fiddle::TYPE_CHAR Immediate with special handling of true/false

- class Bool < Immediate.define(-Fiddle::TYPE_CHAR)

- # Dereference

- def *

- return nil if @addr == 0

- super != 0

- end

-

- def self.[]=(addr, value)

- super(addr, value ? 1 : 0)

- end

- end

-

- # Basically Immediate but without #* to skip auto-dereference of structs.

- class Array

- attr_reader :type

-

- # @param addr [Integer]

- # @param type [Class] RubyVM::RJIT::CType::*

- def initialize(addr, type)

- @addr = addr

- @type = type

- end

-

- # Array access

- def [](index)

- @type.new(@addr)[index]

- end

-

- # Array set

- # @param index [Integer]

- # @param value [Integer, RubyVM::RJIT::CPointer::Struct] an address itself or an object that return an address with to_i

- def []=(index, value)

- @type.new(@addr)[index] = value

- end

-

- private

-

- def self.define(block)

- Class.new(self) do

- define_method(:initialize) do |addr|

- super(addr, block.call)

- end

- end

- end

- end

-

- class Pointer

- attr_reader :type

-

- # @param addr [Integer]

- # @param type [Class] RubyVM::RJIT::CType::*

- def initialize(addr, type)

- @addr = addr

- @type = type

- end

-

- # Move addr to addess this pointer like an array

- def +(index)

- raise ArgumentError unless index.is_a?(Integer)

- Pointer.new(@addr + index * Fiddle::SIZEOF_VOIDP, @type)

- end

-

- # Dereference

- def *

- return nil if dest_addr == 0

- @type.new(dest_addr)

- end

-

- # Array access

- def [](index)

- (self + index).*

- end

-

- # Array set

- # @param index [Integer]

- # @param value [Integer, RubyVM::RJIT::CPointer::Struct] an address itself or an object that return an address with to_i

- def []=(index, value)

- Fiddle::Pointer.new(@addr + index * Fiddle::SIZEOF_VOIDP)[0, Fiddle::SIZEOF_VOIDP] =

- [value.to_i].pack(Fiddle::PackInfo::PACK_MAP[Fiddle::TYPE_VOIDP])

- end

-

- # Get a raw address

- def to_i

- @addr

- end

-

- private

-

- def dest_addr

- Fiddle::Pointer.new(@addr)[0, Fiddle::SIZEOF_VOIDP].unpack1(Fiddle::PackInfo::PACK_MAP[Fiddle::TYPE_VOIDP])

- end

-

- def self.define(block)

- Class.new(self) do

- define_method(:initialize) do |addr|

- super(addr, block.call)

- end

-

- # Type-level []=: Used by struct fields

- # @param addr [Integer]

- # @param value [Integer, RubyVM::RJIT::CPointer::Struct] an address itself, or an object that return an address with to_i

- define_singleton_method(:[]=) do |addr, value|

- value = value.to_i

- Fiddle::Pointer.new(addr)[0, Fiddle::SIZEOF_VOIDP] = [value].pack(Fiddle::PackInfo::PACK_MAP[Fiddle::TYPE_VOIDP])

- end

- end

- end

- end

-

- class BitField

- # @param addr [Integer]

- # @param width [Integer]

- # @param offset [Integer]

- def initialize(addr, width, offset)

- @addr = addr

- @width = width

- @offset = offset

- end

-

- # Dereference

- def *

- byte = Fiddle::Pointer.new(@addr)[0, Fiddle::SIZEOF_CHAR].unpack('c').first

- if @width == 1

- bit = (1 & (byte >> @offset))

- bit == 1

- elsif @width <= 8 && @offset == 0

- bitmask = @width.times.map { |i| 1 << i }.sum

- byte & bitmask

- else

- raise NotImplementedError.new("not-implemented bit field access: width=#{@width} offset=#{@offset}")

- end

- end

-

- # @param width [Integer]

- # @param offset [Integer]

- def self.define(width, offset)

- Class.new(self) do

- define_method(:initialize) do |addr|

- super(addr, width, offset)

- end

- end

- end

- end

-

- # Give a name to a dynamic CPointer class to see it on inspect

- def self.with_class_name(prefix, name, cache: false, &block)

- return block.call if !name.nil? && name.empty?

-

- # Use a cached result only if cache: true

- class_name = "#{prefix}_#{name}"

- klass =

- if cache && self.const_defined?(class_name)

- self.const_get(class_name)

- else

- block.call

- end

-

- # Give it a name unless it's already defined

- unless self.const_defined?(class_name)

- self.const_set(class_name, klass)

- end

-

- klass

- end

- end

-end

-require 'fiddle'

-require 'fiddle/pack'

-require_relative 'c_pointer'

-

-module RubyVM::RJIT

- module CType

- module Struct

- # @param name [String]

- # @param members [Hash{ Symbol => [Integer, RubyVM::RJIT::CType::*] }]

- def self.new(name, sizeof, **members)

- name = members.keys.join('_') if name.empty?

- CPointer.with_class_name('Struct', name) do

- CPointer::Struct.define(sizeof, members)

- end

- end

- end

-

- module Union

- # @param name [String]

- # @param members [Hash{ Symbol => RubyVM::RJIT::CType::* }]

- def self.new(name, sizeof, **members)

- name = members.keys.join('_') if name.empty?

- CPointer.with_class_name('Union', name) do

- CPointer::Union.define(sizeof, members)

- end

- end

- end

-

- module Immediate

- # @param fiddle_type [Integer]

- def self.new(fiddle_type)

- name = Fiddle.constants.find do |const|

- const.start_with?('TYPE_') && Fiddle.const_get(const) == fiddle_type.abs

- end&.name

- name = name.delete_prefix('TYPE_')

- if fiddle_type.negative?

- name.prepend('U')

- end

- CPointer.with_class_name('Immediate', name, cache: true) do

- CPointer::Immediate.define(fiddle_type)

- end

- end

-

- # @param type [String]

- def self.parse(ctype)

- new(Fiddle::Importer.parse_ctype(ctype))

- end

-

- def self.find(size, signed)

- fiddle_type = TYPE_MAP.fetch(size)

- fiddle_type = -fiddle_type unless signed

- new(fiddle_type)

- end

-

- TYPE_MAP = Fiddle::PackInfo::SIZE_MAP.map { |type, size| [size, type.abs] }.to_h

- private_constant :TYPE_MAP

- end

-

- module Bool

- def self.new

- CPointer::Bool

- end

- end

-

- class Array

- def self.new(&block)

- CPointer.with_class_name('Array', block.object_id.to_s) do

- CPointer::Array.define(block)

- end

- end

- end

-

- class Pointer

- # This takes a block to avoid "stack level too deep" on a cyclic reference

- # @param block [Proc]

- def self.new(&block)

- CPointer.with_class_name('Pointer', block.object_id.to_s) do

- CPointer::Pointer.define(block)

- end

- end

- end

-

- module BitField

- # @param width [Integer]

- # @param offset [Integer]

- def self.new(width, offset)

- CPointer.with_class_name('BitField', "#{offset}_#{width}") do

- CPointer::BitField.define(width, offset)

- end

- end

- end

-

- # Types that are referenced but not part of code generation targets

- Stub = ::Struct.new(:name)

-

- # Types that it failed to figure out from the header

- Unknown = Module.new

- end

-end

-module RubyVM::RJIT

- class CodeBlock

- # @param mem_block [Integer] JIT buffer address

- # @param mem_size [Integer] JIT buffer size

- # @param outliend [TrueClass,FalseClass] true for outlined CodeBlock

- def initialize(mem_block:, mem_size:, outlined: false)

- @comments = Hash.new { |h, k| h[k] = [] } if dump_disasm?

- @mem_block = mem_block

- @mem_size = mem_size

- @write_pos = 0

- @outlined = outlined

- end

-

- # @param asm [RubyVM::RJIT::Assembler]

- def write(asm)

- return 0 if @write_pos + asm.size >= @mem_size

-

- start_addr = write_addr

-

- # Write machine code

- C.mprotect_write(@mem_block, @mem_size)

- @write_pos += asm.assemble(start_addr)

- C.mprotect_exec(@mem_block, @mem_size)

-

- end_addr = write_addr

-

- # Convert comment indexes to addresses

- asm.comments.each do |index, comments|

- @comments[start_addr + index] += comments if dump_disasm?

- end

- asm.comments.clear

-

- # Dump disasm if --rjit-dump-disasm

- if C.rjit_opts.dump_disasm && start_addr < end_addr

- dump_disasm(start_addr, end_addr)

- end

- start_addr

- end

-

- def set_write_addr(addr)

- @write_pos = addr - @mem_block

- @comments.delete(addr) if dump_disasm?

- end

-

- def with_write_addr(addr)

- old_write_pos = @write_pos

- set_write_addr(addr)

- yield

- ensure

- @write_pos = old_write_pos

- end

-

- def write_addr

- @mem_block + @write_pos

- end

-

- def include?(addr)

- (@mem_block...(@mem_block + @mem_size)).include?(addr)

- end

-

- def dump_disasm(from, to, io: STDOUT, color: true, test: false)

- C.dump_disasm(from, to, test:).each do |address, mnemonic, op_str|

- @comments.fetch(address, []).each do |comment|

- io.puts colorize(" # #{comment}", bold: true, color:)

- end

- io.puts colorize(" 0x#{format("%x", address)}: #{mnemonic} #{op_str}", color:)

- end

- io.puts

- end

-

- private

-

- def colorize(text, bold: false, color:)

- return text unless color

- buf = +''

- buf << "\e[1m" if bold

- buf << "\e[34m" if @outlined

- buf << text

- buf << "\e[0m"

- buf

- end

-

- def bold(text)

- "\e[1m#{text}\e[0m"

- end

-

- def dump_disasm?

- C.rjit_opts.dump_disasm

- end

- end

-end

-require 'ruby_vm/rjit/assembler'

-require 'ruby_vm/rjit/block'

-require 'ruby_vm/rjit/branch_stub'

-require 'ruby_vm/rjit/code_block'

-require 'ruby_vm/rjit/context'

-require 'ruby_vm/rjit/entry_stub'

-require 'ruby_vm/rjit/exit_compiler'

-require 'ruby_vm/rjit/insn_compiler'

-require 'ruby_vm/rjit/instruction'

-require 'ruby_vm/rjit/invariants'

-require 'ruby_vm/rjit/jit_state'

-require 'ruby_vm/rjit/type'

-

-module RubyVM::RJIT

- # Compilation status

- KeepCompiling = :KeepCompiling

- CantCompile = :CantCompile

- EndBlock = :EndBlock

-

- # Ruby constants

- Qtrue = Fiddle::Qtrue

- Qfalse = Fiddle::Qfalse

- Qnil = Fiddle::Qnil

- Qundef = Fiddle::Qundef

-

- # Callee-saved registers

- # TODO: support using r12/r13 here

- EC = :r14

- CFP = :r15

- SP = :rbx

-

- # Scratch registers: rax, rcx, rdx

-

- # Mark objects in this Array during GC

- GC_REFS = []

-

- # Maximum number of versions per block

- # 1 means always create generic versions

- MAX_VERSIONS = 4

-

- class Compiler

- attr_accessor :write_pos

-

- def self.decode_insn(encoded)

- INSNS.fetch(C.rb_vm_insn_decode(encoded))

- end

-

- def initialize

- mem_size = C.rjit_opts.exec_mem_size * 1024 * 1024

- mem_block = C.mmap(mem_size)

- @cb = CodeBlock.new(mem_block: mem_block, mem_size: mem_size / 2)

- @ocb = CodeBlock.new(mem_block: mem_block + mem_size / 2, mem_size: mem_size / 2, outlined: true)

- @exit_compiler = ExitCompiler.new

- @insn_compiler = InsnCompiler.new(@cb, @ocb, @exit_compiler)

- Invariants.initialize(@cb, @ocb, self, @exit_compiler)

- end

-

- # Compile an ISEQ from its entry point.

- # @param iseq `RubyVM::RJIT::CPointer::Struct_rb_iseq_t`

- # @param cfp `RubyVM::RJIT::CPointer::Struct_rb_control_frame_t`

- def compile(iseq, cfp)

- return unless supported_platform?

- pc = cfp.pc.to_i

- jit = JITState.new(iseq:, cfp:)

- asm = Assembler.new

- compile_prologue(asm, iseq, pc)

- compile_block(asm, jit:, pc:)

- iseq.body.jit_entry = @cb.write(asm)

- rescue Exception => e

- STDERR.puts "#{e.class}: #{e.message}"

- STDERR.puts e.backtrace

- exit 1

- end

-

- # Compile an entry.

- # @param entry [RubyVM::RJIT::EntryStub]

- def entry_stub_hit(entry_stub, cfp)

- # Compile a new entry guard as a next entry

- pc = cfp.pc.to_i

- next_entry = Assembler.new.then do |asm|

- compile_entry_chain_guard(asm, cfp.iseq, pc)

- @cb.write(asm)

- end

-

- # Try to find an existing compiled version of this block

- ctx = Context.new

- block = find_block(cfp.iseq, pc, ctx)

- if block

- # If an existing block is found, generate a jump to the block.

- asm = Assembler.new

- asm.jmp(block.start_addr)

- @cb.write(asm)

- else

- # If this block hasn't yet been compiled, generate blocks after the entry guard.

- asm = Assembler.new

- jit = JITState.new(iseq: cfp.iseq, cfp:)

- compile_block(asm, jit:, pc:, ctx:)

- @cb.write(asm)

-

- block = jit.block

- end

-

- # Regenerate the previous entry

- @cb.with_write_addr(entry_stub.start_addr) do

- # The last instruction of compile_entry_chain_guard is jne

- asm = Assembler.new

- asm.jne(next_entry)

- @cb.write(asm)

- end

-

- return block.start_addr

- rescue Exception => e

- STDERR.puts e.full_message

- exit 1

- end

-

- # Compile a branch stub.

- # @param branch_stub [RubyVM::RJIT::BranchStub]

- # @param cfp `RubyVM::RJIT::CPointer::Struct_rb_control_frame_t`

- # @param target0_p [TrueClass,FalseClass]

- # @return [Integer] The starting address of the compiled branch stub

- def branch_stub_hit(branch_stub, cfp, target0_p)

- # Update cfp->pc for `jit.at_current_insn?`

- target = target0_p ? branch_stub.target0 : branch_stub.target1

- cfp.pc = target.pc

-

- # Reuse an existing block if it already exists

- block = find_block(branch_stub.iseq, target.pc, target.ctx)

-

- # If the branch stub's jump is the last code, allow overwriting part of

- # the old branch code with the new block code.

- fallthrough = block.nil? && @cb.write_addr == branch_stub.end_addr

- if fallthrough

- # If the branch stub's jump is the last code, allow overwriting part of

- # the old branch code with the new block code.

- @cb.set_write_addr(branch_stub.start_addr)

- branch_stub.shape = target0_p ? Next0 : Next1

- Assembler.new.tap do |branch_asm|

- branch_stub.compile.call(branch_asm)

- @cb.write(branch_asm)

- end

- end

-

- # Reuse or generate a block

- if block

- target.address = block.start_addr

- else

- jit = JITState.new(iseq: branch_stub.iseq, cfp:)

- target.address = Assembler.new.then do |asm|

- compile_block(asm, jit:, pc: target.pc, ctx: target.ctx.dup)

- @cb.write(asm)

- end

- block = jit.block

- end

- block.incoming << branch_stub # prepare for invalidate_block

-

- # Re-generate the branch code for non-fallthrough cases

- unless fallthrough

- @cb.with_write_addr(branch_stub.start_addr) do

- branch_asm = Assembler.new

- branch_stub.compile.call(branch_asm)

- @cb.write(branch_asm)

- end

- end

-

- return target.address

- rescue Exception => e

- STDERR.puts e.full_message

- exit 1

- end

-

- # @param iseq `RubyVM::RJIT::CPointer::Struct_rb_iseq_t`

- # @param pc [Integer]

- def invalidate_blocks(iseq, pc)

- list_blocks(iseq, pc).each do |block|

- invalidate_block(block)

- end

-

- # If they were the ISEQ's first blocks, re-compile RJIT entry as well

- if iseq.body.iseq_encoded.to_i == pc

- iseq.body.jit_entry = 0

- iseq.body.jit_entry_calls = 0

- end

- end

-

- def invalidate_block(block)

- iseq = block.iseq

- # Avoid touching GCed ISEQs. We assume it won't be re-entered.

- return unless C.imemo_type_p(iseq, C.imemo_iseq)

-

- # Remove this block from the version array

- remove_block(iseq, block)

-

- # Invalidate the block with entry exit

- unless block.invalidated

- @cb.with_write_addr(block.start_addr) do

- asm = Assembler.new

- asm.comment('invalidate_block')

- asm.jmp(block.entry_exit)

- @cb.write(asm)

- end

- block.invalidated = true

- end

-

- # Re-stub incoming branches

- block.incoming.each do |branch_stub|

- target = [branch_stub.target0, branch_stub.target1].compact.find do |target|

- target.pc == block.pc && target.ctx == block.ctx

- end

- next if target.nil?

- # TODO: Could target.address be a stub address? Is invalidation not needed in that case?

-

- # If the target being re-generated is currently a fallthrough block,

- # the fallthrough code must be rewritten with a jump to the stub.

- if target.address == branch_stub.end_addr

- branch_stub.shape = Default

- end

-

- target.address = Assembler.new.then do |ocb_asm|

- @exit_compiler.compile_branch_stub(block.ctx, ocb_asm, branch_stub, target == branch_stub.target0)

- @ocb.write(ocb_asm)

- end

- @cb.with_write_addr(branch_stub.start_addr) do

- branch_asm = Assembler.new

- branch_stub.compile.call(branch_asm)

- @cb.write(branch_asm)

- end

- end

- end

-

- private

-

- # Callee-saved: rbx, rsp, rbp, r12, r13, r14, r15

- # Caller-saved: rax, rdi, rsi, rdx, rcx, r8, r9, r10, r11

- #

- # @param asm [RubyVM::RJIT::Assembler]

- def compile_prologue(asm, iseq, pc)

- asm.comment('RJIT entry point')

-

- # Save callee-saved registers used by JITed code

- asm.push(CFP)

- asm.push(EC)

- asm.push(SP)

-

- # Move arguments EC and CFP to dedicated registers

- asm.mov(EC, :rdi)

- asm.mov(CFP, :rsi)

-

- # Load sp to a dedicated register

- asm.mov(SP, [CFP, C.rb_control_frame_t.offsetof(:sp)]) # rbx = cfp->sp

-

- # Setup cfp->jit_return

- asm.mov(:rax, leave_exit)

- asm.mov([CFP, C.rb_control_frame_t.offsetof(:jit_return)], :rax)

-

- # We're compiling iseqs that we *expect* to start at `insn_idx`. But in

- # the case of optional parameters, the interpreter can set the pc to a

- # different location depending on the optional parameters. If an iseq

- # has optional parameters, we'll add a runtime check that the PC we've

- # compiled for is the same PC that the interpreter wants us to run with.

- # If they don't match, then we'll take a side exit.

- if iseq.body.param.flags.has_opt

- compile_entry_chain_guard(asm, iseq, pc)

- end

- end

-

- def compile_entry_chain_guard(asm, iseq, pc)

- entry_stub = EntryStub.new

- stub_addr = Assembler.new.then do |ocb_asm|

- @exit_compiler.compile_entry_stub(ocb_asm, entry_stub)

- @ocb.write(ocb_asm)

- end

-

- asm.comment('guard expected PC')

- asm.mov(:rax, pc)

- asm.cmp([CFP, C.rb_control_frame_t.offsetof(:pc)], :rax)

-

- asm.stub(entry_stub) do

- asm.jne(stub_addr)

- end

- end

-

- # @param asm [RubyVM::RJIT::Assembler]

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- def compile_block(asm, jit:, pc:, ctx: Context.new)

- # Mark the block start address and prepare an exit code storage

- ctx = limit_block_versions(jit.iseq, pc, ctx)

- block = Block.new(iseq: jit.iseq, pc:, ctx: ctx.dup)

- jit.block = block

- asm.block(block)

-

- iseq = jit.iseq

- asm.comment("Block: #{iseq.body.location.label}@#{C.rb_iseq_path(iseq)}:#{iseq_lineno(iseq, pc)}")

-

- # Compile each insn

- index = (pc - iseq.body.iseq_encoded.to_i) / C.VALUE.size

- while index < iseq.body.iseq_size

- # Set the current instruction

- insn = self.class.decode_insn(iseq.body.iseq_encoded[index])

- jit.pc = (iseq.body.iseq_encoded + index).to_i

- jit.stack_size_for_pc = ctx.stack_size

- jit.side_exit_for_pc.clear

-

- # If previous instruction requested to record the boundary

- if jit.record_boundary_patch_point

- # Generate an exit to this instruction and record it

- exit_pos = Assembler.new.then do |ocb_asm|

- @exit_compiler.compile_side_exit(jit.pc, ctx, ocb_asm)

- @ocb.write(ocb_asm)

- end

- Invariants.record_global_inval_patch(asm, exit_pos)

- jit.record_boundary_patch_point = false

- end

-

- # In debug mode, verify our existing assumption

- if C.rjit_opts.verify_ctx && jit.at_current_insn?

- verify_ctx(jit, ctx)

- end

-

- case status = @insn_compiler.compile(jit, ctx, asm, insn)

- when KeepCompiling

- # For now, reset the chain depth after each instruction as only the

- # first instruction in the block can concern itself with the depth.

- ctx.chain_depth = 0

-

- index += insn.len

- when EndBlock

- # TODO: pad nops if entry exit exists (not needed for x86_64?)

- break

- when CantCompile

- # Rewind stack_size using ctx.with_stack_size to allow stack_size changes

- # before you return CantCompile.

- @exit_compiler.compile_side_exit(jit.pc, ctx.with_stack_size(jit.stack_size_for_pc), asm)

-

- # If this is the first instruction, this block never needs to be invalidated.

- if block.pc == iseq.body.iseq_encoded.to_i + index * C.VALUE.size

- block.invalidated = true

- end

-

- break

- else

- raise "compiling #{insn.name} returned unexpected status: #{status.inspect}"

- end

- end

-

- incr_counter(:compiled_block_count)

- add_block(iseq, block)

- end

-

- def leave_exit

- @leave_exit ||= Assembler.new.then do |asm|

- @exit_compiler.compile_leave_exit(asm)

- @ocb.write(asm)

- end

- end

-

- def incr_counter(name)

- if C.rjit_opts.stats

- C.rb_rjit_counters[name][0] += 1

- end

- end

-

- # Produce a generic context when the block version limit is hit for the block

- def limit_block_versions(iseq, pc, ctx)

- # Guard chains implement limits separately, do nothing

- if ctx.chain_depth > 0

- return ctx.dup

- end

-

- # If this block version we're about to add will hit the version limit

- if list_blocks(iseq, pc).size + 1 >= MAX_VERSIONS

- # Produce a generic context that stores no type information,

- # but still respects the stack_size and sp_offset constraints.

- # This new context will then match all future requests.

- generic_ctx = Context.new

- generic_ctx.stack_size = ctx.stack_size

- generic_ctx.sp_offset = ctx.sp_offset

-

- if ctx.diff(generic_ctx) == TypeDiff::Incompatible

- raise 'should substitute a compatible context'

- end

-

- return generic_ctx

- end

-

- return ctx.dup

- end

-

- def list_blocks(iseq, pc)

- rjit_blocks(iseq)[pc]

- end

-

- # @param [Integer] pc

- # @param [RubyVM::RJIT::Context] ctx

- # @return [RubyVM::RJIT::Block,NilClass]

- def find_block(iseq, pc, ctx)

- versions = rjit_blocks(iseq)[pc]

-

- best_version = nil

- best_diff = Float::INFINITY

-

- versions.each do |block|

- # Note that we always prefer the first matching

- # version found because of inline-cache chains

- case ctx.diff(block.ctx)

- in TypeDiff::Compatible[diff] if diff < best_diff

- best_version = block

- best_diff = diff

- else

- end

- end

-

- return best_version

- end

-

- # @param [RubyVM::RJIT::Block] block

- def add_block(iseq, block)

- rjit_blocks(iseq)[block.pc] << block

- end

-

- # @param [RubyVM::RJIT::Block] block

- def remove_block(iseq, block)

- rjit_blocks(iseq)[block.pc].delete(block)

- end

-

- def rjit_blocks(iseq)

- # Guard against ISEQ GC at random moments

-

- unless C.imemo_type_p(iseq, C.imemo_iseq)

- return Hash.new { |h, k| h[k] = [] }

- end

-

- unless iseq.body.rjit_blocks

- iseq.body.rjit_blocks = Hash.new { |blocks, pc| blocks[pc] = [] }

- # For some reason, rb_rjit_iseq_mark didn't protect this Hash

- # from being freed. So we rely on GC_REFS to keep the Hash.

- GC_REFS << iseq.body.rjit_blocks

- end

- iseq.body.rjit_blocks

- end

-

- def iseq_lineno(iseq, pc)

- C.rb_iseq_line_no(iseq, (pc - iseq.body.iseq_encoded.to_i) / C.VALUE.size)

- rescue RangeError # bignum too big to convert into `unsigned long long' (RangeError)

- -1

- end

-

- # Verify the ctx's types and mappings against the compile-time stack, self, and locals.

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- def verify_ctx(jit, ctx)

- # Only able to check types when at current insn

- assert(jit.at_current_insn?)

-

- self_val = jit.peek_at_self

- self_val_type = Type.from(self_val)

-

- # Verify self operand type

- assert_compatible(self_val_type, ctx.get_opnd_type(SelfOpnd))

-

- # Verify stack operand types

- [ctx.stack_size, MAX_TEMP_TYPES].min.times do |i|

- learned_mapping, learned_type = ctx.get_opnd_mapping(StackOpnd[i])

- stack_val = jit.peek_at_stack(i)

- val_type = Type.from(stack_val)

-

- case learned_mapping

- in MapToSelf

- if C.to_value(self_val) != C.to_value(stack_val)

- raise "verify_ctx: stack value was mapped to self, but values did not match:\n"\

- "stack: #{stack_val.inspect}, self: #{self_val.inspect}"

- end

- in MapToLocal[local_idx]

- local_val = jit.peek_at_local(local_idx)

- if C.to_value(local_val) != C.to_value(stack_val)

- raise "verify_ctx: stack value was mapped to local, but values did not match:\n"\

- "stack: #{stack_val.inspect}, local: #{local_val.inspect}"

- end

- in MapToStack

- # noop

- end

-

- # If the actual type differs from the learned type

- assert_compatible(val_type, learned_type)

- end

-

- # Verify local variable types

- local_table_size = jit.iseq.body.local_table_size

- [local_table_size, MAX_TEMP_TYPES].min.times do |i|

- learned_type = ctx.get_local_type(i)

- local_val = jit.peek_at_local(i)

- local_type = Type.from(local_val)

-

- assert_compatible(local_type, learned_type)

- end

- end

-

- def assert_compatible(actual_type, ctx_type)

- if actual_type.diff(ctx_type) == TypeDiff::Incompatible

- raise "verify_ctx: ctx type (#{ctx_type.type.inspect}) is incompatible with actual type (#{actual_type.type.inspect})"

- end

- end

-

- def assert(cond)

- unless cond

- raise "'#{cond.inspect}' was not true"

- end

- end

-

- def supported_platform?

- return @supported_platform if defined?(@supported_platform)

- @supported_platform = RUBY_PLATFORM.match?(/x86_64/).tap do |supported|

- warn "warning: RJIT does not support #{RUBY_PLATFORM} yet" unless supported

- end

- end

- end

-end

-module RubyVM::RJIT

- # Maximum number of temp value types we keep track of

- MAX_TEMP_TYPES = 8

- # Maximum number of local variable types we keep track of

- MAX_LOCAL_TYPES = 8

-

- # Operand to a YARV bytecode instruction

- SelfOpnd = :SelfOpnd # The value is self

- StackOpnd = Data.define(:index) # Temporary stack operand with stack index

-

- # Potential mapping of a value on the temporary stack to self,

- # a local variable, or constant so that we can track its type

- MapToStack = :MapToStack # Normal stack value

- MapToSelf = :MapToSelf # Temp maps to the self operand

- MapToLocal = Data.define(:local_index) # Temp maps to a local variable with index

-

- class Context < Struct.new(

- :stack_size, # @param [Integer] The number of values on the stack

- :sp_offset, # @param [Integer] JIT sp offset relative to the interpreter's sp

- :chain_depth, # @param [Integer] jit_chain_guard depth

- :local_types, # @param [Array<RubyVM::RJIT::Type>] Local variable types we keep track of

- :temp_types, # @param [Array<RubyVM::RJIT::Type>] Temporary variable types we keep track of

- :self_type, # @param [RubyVM::RJIT::Type] Type we track for self

- :temp_mapping, # @param [Array<Symbol>] Mapping of temp stack entries to types we track

- )

- def initialize(

- stack_size: 0,

- sp_offset: 0,

- chain_depth: 0,

- local_types: [Type::Unknown] * MAX_LOCAL_TYPES,

- temp_types: [Type::Unknown] * MAX_TEMP_TYPES,

- self_type: Type::Unknown,

- temp_mapping: [MapToStack] * MAX_TEMP_TYPES

- ) = super

-

- # Deep dup by default for safety

- def dup

- ctx = super

- ctx.local_types = ctx.local_types.dup

- ctx.temp_types = ctx.temp_types.dup

- ctx.temp_mapping = ctx.temp_mapping.dup

- ctx

- end

-

- # Create a new Context instance with a given stack_size and sp_offset adjusted

- # accordingly. This is useful when you want to virtually rewind a stack_size for

- # generating a side exit while considering past sp_offset changes on gen_save_sp.

- def with_stack_size(stack_size)

- ctx = self.dup

- ctx.sp_offset -= ctx.stack_size - stack_size

- ctx.stack_size = stack_size

- ctx

- end

-

- def stack_opnd(depth_from_top)

- [SP, C.VALUE.size * (self.sp_offset - 1 - depth_from_top)]

- end

-

- def sp_opnd(offset_bytes = 0)

- [SP, (C.VALUE.size * self.sp_offset) + offset_bytes]

- end

-

- # Push one new value on the temp stack with an explicit mapping

- # Return a pointer to the new stack top

- def stack_push_mapping(mapping_temp_type)

- stack_size = self.stack_size

-

- # Keep track of the type and mapping of the value

- if stack_size < MAX_TEMP_TYPES

- mapping, temp_type = mapping_temp_type

- self.temp_mapping[stack_size] = mapping

- self.temp_types[stack_size] = temp_type

-

- case mapping

- in MapToLocal[idx]

- assert(idx < MAX_LOCAL_TYPES)

- else

- end

- end

-

- self.stack_size += 1

- self.sp_offset += 1

-

- return self.stack_opnd(0)

- end

-

- # Push one new value on the temp stack

- # Return a pointer to the new stack top

- def stack_push(val_type)

- return self.stack_push_mapping([MapToStack, val_type])

- end

-

- # Push the self value on the stack

- def stack_push_self

- return self.stack_push_mapping([MapToStack, Type::Unknown])

- end

-

- # Push a local variable on the stack

- def stack_push_local(local_idx)

- if local_idx >= MAX_LOCAL_TYPES

- return self.stack_push(Type::Unknown)

- end

-

- return self.stack_push_mapping([MapToLocal[local_idx], Type::Unknown])

- end

-

- # Pop N values off the stack

- # Return a pointer to the stack top before the pop operation

- def stack_pop(n = 1)

- assert(n <= self.stack_size)

-

- top = self.stack_opnd(0)

-

- # Clear the types of the popped values

- n.times do |i|

- idx = self.stack_size - i - 1

-

- if idx < MAX_TEMP_TYPES

- self.temp_types[idx] = Type::Unknown

- self.temp_mapping[idx] = MapToStack

- end

- end

-

- self.stack_size -= n

- self.sp_offset -= n

-

- return top

- end

-

- def shift_stack(argc)

- assert(argc < self.stack_size)

-

- method_name_index = self.stack_size - argc - 1

-

- (method_name_index...(self.stack_size - 1)).each do |i|

- if i + 1 < MAX_TEMP_TYPES

- self.temp_types[i] = self.temp_types[i + 1]

- self.temp_mapping[i] = self.temp_mapping[i + 1]

- end

- end

- self.stack_pop(1)

- end

-

- # Get the type of an instruction operand

- def get_opnd_type(opnd)

- case opnd

- in SelfOpnd

- self.self_type

- in StackOpnd[idx]

- assert(idx < self.stack_size)

- stack_idx = self.stack_size - 1 - idx

-

- # If outside of tracked range, do nothing

- if stack_idx >= MAX_TEMP_TYPES

- return Type::Unknown

- end

-

- mapping = self.temp_mapping[stack_idx]

-

- case mapping

- in MapToSelf

- self.self_type

- in MapToStack

- self.temp_types[self.stack_size - 1 - idx]

- in MapToLocal[idx]

- assert(idx < MAX_LOCAL_TYPES)

- self.local_types[idx]

- end

- end

- end

-

- # Get the currently tracked type for a local variable

- def get_local_type(idx)

- self.local_types[idx] || Type::Unknown

- end

-

- # Upgrade (or "learn") the type of an instruction operand

- # This value must be compatible and at least as specific as the previously known type.

- # If this value originated from self, or an lvar, the learned type will be

- # propagated back to its source.

- def upgrade_opnd_type(opnd, opnd_type)

- case opnd

- in SelfOpnd

- self.self_type = self.self_type.upgrade(opnd_type)

- in StackOpnd[idx]

- assert(idx < self.stack_size)

- stack_idx = self.stack_size - 1 - idx

-

- # If outside of tracked range, do nothing

- if stack_idx >= MAX_TEMP_TYPES

- return

- end

-

- mapping = self.temp_mapping[stack_idx]

-

- case mapping

- in MapToSelf

- self.self_type = self.self_type.upgrade(opnd_type)

- in MapToStack

- self.temp_types[stack_idx] = self.temp_types[stack_idx].upgrade(opnd_type)

- in MapToLocal[idx]

- assert(idx < MAX_LOCAL_TYPES)

- self.local_types[idx] = self.local_types[idx].upgrade(opnd_type)

- end

- end

- end

-

- # Get both the type and mapping (where the value originates) of an operand.

- # This is can be used with stack_push_mapping or set_opnd_mapping to copy

- # a stack value's type while maintaining the mapping.

- def get_opnd_mapping(opnd)

- opnd_type = self.get_opnd_type(opnd)

-

- case opnd

- in SelfOpnd

- return [MapToSelf, opnd_type]

- in StackOpnd[idx]

- assert(idx < self.stack_size)

- stack_idx = self.stack_size - 1 - idx

-

- if stack_idx < MAX_TEMP_TYPES

- return [self.temp_mapping[stack_idx], opnd_type]

- else

- # We can't know the source of this stack operand, so we assume it is

- # a stack-only temporary. type will be UNKNOWN

- assert(opnd_type == Type::Unknown)

- return [MapToStack, opnd_type]

- end

- end

- end

-

- # Overwrite both the type and mapping of a stack operand.

- def set_opnd_mapping(opnd, mapping_opnd_type)

- case opnd

- in SelfOpnd

- raise 'self always maps to self'

- in StackOpnd[idx]

- assert(idx < self.stack_size)

- stack_idx = self.stack_size - 1 - idx

-

- # If outside of tracked range, do nothing

- if stack_idx >= MAX_TEMP_TYPES

- return

- end

-

- mapping, opnd_type = mapping_opnd_type

- self.temp_mapping[stack_idx] = mapping

-

- # Only used when mapping == MAP_STACK

- self.temp_types[stack_idx] = opnd_type

- end

- end

-

- # Set the type of a local variable

- def set_local_type(local_idx, local_type)

- if local_idx >= MAX_LOCAL_TYPES

- return

- end

-

- # If any values on the stack map to this local we must detach them

- MAX_TEMP_TYPES.times do |stack_idx|

- case self.temp_mapping[stack_idx]

- in MapToStack

- # noop

- in MapToSelf

- # noop

- in MapToLocal[idx]

- if idx == local_idx

- self.temp_types[stack_idx] = self.local_types[idx]

- self.temp_mapping[stack_idx] = MapToStack

- else

- # noop

- end

- end

- end

-

- self.local_types[local_idx] = local_type

- end

-

- # Erase local variable type information

- # eg: because of a call we can't track

- def clear_local_types

- # When clearing local types we must detach any stack mappings to those

- # locals. Even if local values may have changed, stack values will not.

- MAX_TEMP_TYPES.times do |stack_idx|

- case self.temp_mapping[stack_idx]

- in MapToStack

- # noop

- in MapToSelf

- # noop

- in MapToLocal[local_idx]

- self.temp_types[stack_idx] = self.local_types[local_idx]

- self.temp_mapping[stack_idx] = MapToStack

- end

- end

-

- # Clear the local types

- self.local_types = [Type::Unknown] * MAX_LOCAL_TYPES

- end

-

- # Compute a difference score for two context objects

- def diff(dst)

- # Self is the source context (at the end of the predecessor)

- src = self

-

- # Can only lookup the first version in the chain

- if dst.chain_depth != 0

- return TypeDiff::Incompatible

- end

-

- # Blocks with depth > 0 always produce new versions

- # Sidechains cannot overlap

- if src.chain_depth != 0

- return TypeDiff::Incompatible

- end

-

- if dst.stack_size != src.stack_size

- return TypeDiff::Incompatible

- end

-

- if dst.sp_offset != src.sp_offset

- return TypeDiff::Incompatible

- end

-

- # Difference sum

- diff = 0

-

- # Check the type of self

- diff += case src.self_type.diff(dst.self_type)

- in TypeDiff::Compatible[diff] then diff

- in TypeDiff::Incompatible then return TypeDiff::Incompatible

- end

-

- # For each local type we track

- src.local_types.size.times do |i|

- t_src = src.local_types[i]

- t_dst = dst.local_types[i]

- diff += case t_src.diff(t_dst)

- in TypeDiff::Compatible[diff] then diff

- in TypeDiff::Incompatible then return TypeDiff::Incompatible

- end

- end

-

- # For each value on the temp stack

- src.stack_size.times do |i|

- src_mapping, src_type = src.get_opnd_mapping(StackOpnd[i])

- dst_mapping, dst_type = dst.get_opnd_mapping(StackOpnd[i])

-

- # If the two mappings aren't the same

- if src_mapping != dst_mapping

- if dst_mapping == MapToStack

- # We can safely drop information about the source of the temp

- # stack operand.

- diff += 1

- else

- return TypeDiff::Incompatible

- end

- end

-

- diff += case src_type.diff(dst_type)

- in TypeDiff::Compatible[diff] then diff

- in TypeDiff::Incompatible then return TypeDiff::Incompatible

- end

- end

-

- return TypeDiff::Compatible[diff]

- end

-

- private

-

- def assert(cond)

- unless cond

- raise "'#{cond.inspect}' was not true"

- end

- end

- end

-end

-module RubyVM::RJIT

- class EntryStub < Struct.new(

- :start_addr, # @param [Integer] Stub source start address to be re-generated

- :end_addr, # @param [Integer] Stub source end address to be re-generated

- )

- end

-end

-module RubyVM::RJIT

- class ExitCompiler

- def initialize = freeze

-

- # Used for invalidating a block on entry.

- # @param pc [Integer]

- # @param asm [RubyVM::RJIT::Assembler]

- def compile_entry_exit(pc, ctx, asm, cause:)

- # Fix pc/sp offsets for the interpreter

- save_pc_and_sp(pc, ctx, asm, reset_sp_offset: false)

-

- # Increment per-insn exit counter

- count_insn_exit(pc, asm)

-

- # Restore callee-saved registers

- asm.comment("#{cause}: entry exit")

- asm.pop(SP)

- asm.pop(EC)

- asm.pop(CFP)

-

- asm.mov(C_RET, Qundef)

- asm.ret

- end

-

- # Set to cfp->jit_return by default for leave insn

- # @param asm [RubyVM::RJIT::Assembler]

- def compile_leave_exit(asm)

- asm.comment('default cfp->jit_return')

-

- # Restore callee-saved registers

- asm.pop(SP)

- asm.pop(EC)

- asm.pop(CFP)

-

- # :rax is written by #leave

- asm.ret

- end

-

- # Fire cfunc events on invalidation by TracePoint

- # @param asm [RubyVM::RJIT::Assembler]

- def compile_full_cfunc_return(asm)

- # This chunk of code expects REG_EC to be filled properly and

- # RAX to contain the return value of the C method.

-

- asm.comment('full cfunc return')

- asm.mov(C_ARGS[0], EC)

- asm.mov(C_ARGS[1], :rax)

- asm.call(C.rjit_full_cfunc_return)

-

- # TODO: count the exit

-

- # Restore callee-saved registers

- asm.pop(SP)

- asm.pop(EC)

- asm.pop(CFP)

-

- asm.mov(C_RET, Qundef)

- asm.ret

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def compile_side_exit(pc, ctx, asm)

- # Fix pc/sp offsets for the interpreter

- save_pc_and_sp(pc, ctx.dup, asm) # dup to avoid sp_offset update

-

- # Increment per-insn exit counter

- count_insn_exit(pc, asm)

-

- # Restore callee-saved registers

- asm.comment("exit to interpreter on #{pc_to_insn(pc).name}")

- asm.pop(SP)

- asm.pop(EC)

- asm.pop(CFP)

-

- asm.mov(C_RET, Qundef)

- asm.ret

- end

-

- # @param asm [RubyVM::RJIT::Assembler]

- # @param entry_stub [RubyVM::RJIT::EntryStub]

- def compile_entry_stub(asm, entry_stub)

- # Call rb_rjit_entry_stub_hit

- asm.comment('entry stub hit')

- asm.mov(C_ARGS[0], to_value(entry_stub))

- asm.call(C.rb_rjit_entry_stub_hit)

-

- # Jump to the address returned by rb_rjit_entry_stub_hit

- asm.jmp(:rax)

- end

-

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- # @param branch_stub [RubyVM::RJIT::BranchStub]

- # @param target0_p [TrueClass,FalseClass]

- def compile_branch_stub(ctx, asm, branch_stub, target0_p)

- # Call rb_rjit_branch_stub_hit

- iseq = branch_stub.iseq

- if C.rjit_opts.dump_disasm && C.imemo_type_p(iseq, C.imemo_iseq) # Guard against ISEQ GC at random moments

- asm.comment("branch stub hit: #{iseq.body.location.label}@#{C.rb_iseq_path(iseq)}:#{iseq_lineno(iseq, target0_p ? branch_stub.target0.pc : branch_stub.target1.pc)}")

- end

- asm.mov(:rdi, to_value(branch_stub))

- asm.mov(:esi, ctx.sp_offset)

- asm.mov(:edx, target0_p ? 1 : 0)

- asm.call(C.rb_rjit_branch_stub_hit)

-

- # Jump to the address returned by rb_rjit_branch_stub_hit

- asm.jmp(:rax)

- end

-

- private

-

- def pc_to_insn(pc)

- Compiler.decode_insn(C.VALUE.new(pc).*)

- end

-

- # @param pc [Integer]

- # @param asm [RubyVM::RJIT::Assembler]

- def count_insn_exit(pc, asm)

- if C.rjit_opts.stats

- insn = Compiler.decode_insn(C.VALUE.new(pc).*)

- asm.comment("increment insn exit: #{insn.name}")

- asm.mov(:rax, (C.rjit_insn_exits + insn.bin).to_i)

- asm.add([:rax], 1) # TODO: lock

- end

- if C.rjit_opts.trace_exits

- asm.comment('rjit_record_exit_stack')

- asm.mov(C_ARGS[0], pc)

- asm.call(C.rjit_record_exit_stack)

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def save_pc_and_sp(pc, ctx, asm, reset_sp_offset: true)

- # Update pc (TODO: manage PC offset?)

- asm.comment("save PC#{' and SP' if ctx.sp_offset != 0} to CFP")

- asm.mov(:rax, pc) # rax = jit.pc

- asm.mov([CFP, C.rb_control_frame_t.offsetof(:pc)], :rax) # cfp->pc = rax

-

- # Update sp

- if ctx.sp_offset != 0

- asm.add(SP, C.VALUE.size * ctx.sp_offset) # sp += stack_size

- asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], SP) # cfp->sp = sp

- if reset_sp_offset

- ctx.sp_offset = 0

- end

- end

- end

-

- def to_value(obj)

- GC_REFS << obj

- C.to_value(obj)

- end

-

- def iseq_lineno(iseq, pc)

- C.rb_iseq_line_no(iseq, (pc - iseq.body.iseq_encoded.to_i) / C.VALUE.size)

- rescue RangeError # bignum too big to convert into `unsigned long long' (RangeError)

- -1

- end

- end

-end

-module RubyVM::RJIT

- module Hooks # :nodoc: all

- def self.on_bop_redefined(_redefined_flag, _bop)

- # C.rjit_cancel_all("BOP is redefined")

- end

-

- def self.on_cme_invalidate(cme)

- cme = C.rb_callable_method_entry_struct.new(cme)

- Invariants.on_cme_invalidate(cme)

- end

-

- def self.on_ractor_spawn

- # C.rjit_cancel_all("Ractor is spawned")

- end

-

- # Global constant changes like const_set

- def self.on_constant_state_changed(id)

- Invariants.on_constant_state_changed(id)

- end

-

- # ISEQ-specific constant invalidation

- def self.on_constant_ic_update(iseq, ic, insn_idx)

- iseq = C.rb_iseq_t.new(iseq)

- ic = C.IC.new(ic)

- Invariants.on_constant_ic_update(iseq, ic, insn_idx)

- end

-

- def self.on_tracing_invalidate_all(_new_iseq_events)

- Invariants.on_tracing_invalidate_all

- end

-

- def self.on_update_references

- Invariants.on_update_references

- end

- end

-end

-# frozen_string_literal: true

-module RubyVM::RJIT

- class InsnCompiler

- # struct rb_calling_info. Storing flags instead of ci.

- CallingInfo = Struct.new(:argc, :flags, :kwarg, :ci_addr, :send_shift, :block_handler) do

- def kw_splat = flags & C::VM_CALL_KW_SPLAT != 0

- end

-

- # @param ocb [CodeBlock]

- # @param exit_compiler [RubyVM::RJIT::ExitCompiler]

- def initialize(cb, ocb, exit_compiler)

- @ocb = ocb

- @exit_compiler = exit_compiler

-

- @cfunc_codegen_table = {}

- register_cfunc_codegen_funcs

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- # @param insn `RubyVM::RJIT::Instruction`

- def compile(jit, ctx, asm, insn)

- asm.incr_counter(:rjit_insns_count)

-

- stack = ctx.stack_size.times.map do |stack_idx|

- ctx.get_opnd_type(StackOpnd[ctx.stack_size - stack_idx - 1]).type

- end

- locals = jit.iseq.body.local_table_size.times.map do |local_idx|

- (ctx.local_types[local_idx] || Type::Unknown).type

- end

-

- insn_idx = format('%04d', (jit.pc.to_i - jit.iseq.body.iseq_encoded.to_i) / C.VALUE.size)

- asm.comment("Insn: #{insn_idx} #{insn.name} (stack: [#{stack.join(', ')}], locals: [#{locals.join(', ')}])")

-

- # 83/102

- case insn.name

- when :nop then nop(jit, ctx, asm)

- when :getlocal then getlocal(jit, ctx, asm)

- when :setlocal then setlocal(jit, ctx, asm)

- when :getblockparam then getblockparam(jit, ctx, asm)

- # setblockparam

- when :getblockparamproxy then getblockparamproxy(jit, ctx, asm)

- when :getspecial then getspecial(jit, ctx, asm)

- # setspecial

- when :getinstancevariable then getinstancevariable(jit, ctx, asm)

- when :setinstancevariable then setinstancevariable(jit, ctx, asm)

- when :getclassvariable then getclassvariable(jit, ctx, asm)

- when :setclassvariable then setclassvariable(jit, ctx, asm)

- when :opt_getconstant_path then opt_getconstant_path(jit, ctx, asm)

- when :getconstant then getconstant(jit, ctx, asm)

- # setconstant

- when :getglobal then getglobal(jit, ctx, asm)

- # setglobal

- when :putnil then putnil(jit, ctx, asm)

- when :putself then putself(jit, ctx, asm)

- when :putobject then putobject(jit, ctx, asm)

- when :putspecialobject then putspecialobject(jit, ctx, asm)

- when :putstring then putstring(jit, ctx, asm)

- when :putchilledstring then putchilledstring(jit, ctx, asm)

- when :concatstrings then concatstrings(jit, ctx, asm)

- when :anytostring then anytostring(jit, ctx, asm)

- when :toregexp then toregexp(jit, ctx, asm)

- when :intern then intern(jit, ctx, asm)

- when :newarray then newarray(jit, ctx, asm)

- when :duparray then duparray(jit, ctx, asm)

- # duphash

- when :expandarray then expandarray(jit, ctx, asm)

- when :concatarray then concatarray(jit, ctx, asm)

- when :splatarray then splatarray(jit, ctx, asm)

- when :newhash then newhash(jit, ctx, asm)

- when :newrange then newrange(jit, ctx, asm)

- when :pop then pop(jit, ctx, asm)

- when :dup then dup(jit, ctx, asm)

- when :dupn then dupn(jit, ctx, asm)

- when :swap then swap(jit, ctx, asm)

- # opt_reverse

- when :topn then topn(jit, ctx, asm)

- when :setn then setn(jit, ctx, asm)

- when :adjuststack then adjuststack(jit, ctx, asm)

- when :defined then defined(jit, ctx, asm)

- when :definedivar then definedivar(jit, ctx, asm)

- # checkmatch

- when :checkkeyword then checkkeyword(jit, ctx, asm)

- # checktype

- # defineclass

- # definemethod

- # definesmethod

- when :send then send(jit, ctx, asm)

- when :opt_send_without_block then opt_send_without_block(jit, ctx, asm)

- when :objtostring then objtostring(jit, ctx, asm)

- when :opt_str_freeze then opt_str_freeze(jit, ctx, asm)

- when :opt_ary_freeze then opt_ary_freeze(jit, ctx, asm)

- when :opt_hash_freeze then opt_hash_freeze(jit, ctx, asm)

- when :opt_nil_p then opt_nil_p(jit, ctx, asm)

- # opt_str_uminus

- when :opt_newarray_send then opt_newarray_send(jit, ctx, asm)

- when :invokesuper then invokesuper(jit, ctx, asm)

- when :invokeblock then invokeblock(jit, ctx, asm)

- when :leave then leave(jit, ctx, asm)

- when :throw then throw(jit, ctx, asm)

- when :jump then jump(jit, ctx, asm)

- when :branchif then branchif(jit, ctx, asm)

- when :branchunless then branchunless(jit, ctx, asm)

- when :branchnil then branchnil(jit, ctx, asm)

- # once

- when :opt_case_dispatch then opt_case_dispatch(jit, ctx, asm)

- when :opt_plus then opt_plus(jit, ctx, asm)

- when :opt_minus then opt_minus(jit, ctx, asm)

- when :opt_mult then opt_mult(jit, ctx, asm)

- when :opt_div then opt_div(jit, ctx, asm)

- when :opt_mod then opt_mod(jit, ctx, asm)

- when :opt_eq then opt_eq(jit, ctx, asm)

- when :opt_neq then opt_neq(jit, ctx, asm)

- when :opt_lt then opt_lt(jit, ctx, asm)

- when :opt_le then opt_le(jit, ctx, asm)

- when :opt_gt then opt_gt(jit, ctx, asm)

- when :opt_ge then opt_ge(jit, ctx, asm)

- when :opt_ltlt then opt_ltlt(jit, ctx, asm)

- when :opt_and then opt_and(jit, ctx, asm)

- when :opt_or then opt_or(jit, ctx, asm)

- when :opt_aref then opt_aref(jit, ctx, asm)

- when :opt_aset then opt_aset(jit, ctx, asm)

- # opt_aset_with

- # opt_aref_with

- when :opt_length then opt_length(jit, ctx, asm)

- when :opt_size then opt_size(jit, ctx, asm)

- when :opt_empty_p then opt_empty_p(jit, ctx, asm)

- when :opt_succ then opt_succ(jit, ctx, asm)

- when :opt_not then opt_not(jit, ctx, asm)

- when :opt_regexpmatch2 then opt_regexpmatch2(jit, ctx, asm)

- # invokebuiltin

- when :opt_invokebuiltin_delegate then opt_invokebuiltin_delegate(jit, ctx, asm)

- when :opt_invokebuiltin_delegate_leave then opt_invokebuiltin_delegate_leave(jit, ctx, asm)

- when :getlocal_WC_0 then getlocal_WC_0(jit, ctx, asm)

- when :getlocal_WC_1 then getlocal_WC_1(jit, ctx, asm)

- when :setlocal_WC_0 then setlocal_WC_0(jit, ctx, asm)

- when :setlocal_WC_1 then setlocal_WC_1(jit, ctx, asm)

- when :putobject_INT2FIX_0_ then putobject_INT2FIX_0_(jit, ctx, asm)

- when :putobject_INT2FIX_1_ then putobject_INT2FIX_1_(jit, ctx, asm)

- else CantCompile

- end

- end

-

- private

-

- #

- # Insns

- #

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def nop(jit, ctx, asm)

- # Do nothing

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def getlocal(jit, ctx, asm)

- idx = jit.operand(0)

- level = jit.operand(1)

- jit_getlocal_generic(jit, ctx, asm, idx:, level:)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def getlocal_WC_0(jit, ctx, asm)

- idx = jit.operand(0)

- jit_getlocal_generic(jit, ctx, asm, idx:, level: 0)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def getlocal_WC_1(jit, ctx, asm)

- idx = jit.operand(0)

- jit_getlocal_generic(jit, ctx, asm, idx:, level: 1)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def setlocal(jit, ctx, asm)

- idx = jit.operand(0)

- level = jit.operand(1)

- jit_setlocal_generic(jit, ctx, asm, idx:, level:)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def setlocal_WC_0(jit, ctx, asm)

- idx = jit.operand(0)

- jit_setlocal_generic(jit, ctx, asm, idx:, level: 0)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def setlocal_WC_1(jit, ctx, asm)

- idx = jit.operand(0)

- jit_setlocal_generic(jit, ctx, asm, idx:, level: 1)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def getblockparam(jit, ctx, asm)

- # EP level

- level = jit.operand(1)

-

- # Save the PC and SP because we might allocate

- jit_prepare_routine_call(jit, ctx, asm)

-

- # A mirror of the interpreter code. Checking for the case

- # where it's pushing rb_block_param_proxy.

- side_exit = side_exit(jit, ctx)

-

- # Load environment pointer EP from CFP

- ep_reg = :rax

- jit_get_ep(asm, level, reg: ep_reg)

-

- # Bail when VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) is non zero

- # FIXME: This is testing bits in the same place that the WB check is testing.

- # We should combine these at some point

- asm.test([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], C::VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM)

-

- # If the frame flag has been modified, then the actual proc value is

- # already in the EP and we should just use the value.

- frame_flag_modified = asm.new_label('frame_flag_modified')

- asm.jnz(frame_flag_modified)

-

- # This instruction writes the block handler to the EP. If we need to

- # fire a write barrier for the write, then exit (we'll let the

- # interpreter handle it so it can fire the write barrier).

- # flags & VM_ENV_FLAG_WB_REQUIRED

- asm.test([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], C::VM_ENV_FLAG_WB_REQUIRED)

-

- # if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0

- asm.jnz(side_exit)

-

- # Convert the block handler in to a proc

- # call rb_vm_bh_to_procval(const rb_execution_context_t *ec, VALUE block_handler)

- asm.mov(C_ARGS[0], EC)

- # The block handler for the current frame

- # note, VM_ASSERT(VM_ENV_LOCAL_P(ep))

- asm.mov(C_ARGS[1], [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL])

- asm.call(C.rb_vm_bh_to_procval)

-

- # Load environment pointer EP from CFP (again)

- ep_reg = :rcx

- jit_get_ep(asm, level, reg: ep_reg)

-

- # Write the value at the environment pointer

- idx = jit.operand(0)

- offs = -(C.VALUE.size * idx)

- asm.mov([ep_reg, offs], C_RET);

-

- # Set the frame modified flag

- asm.mov(:rax, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS]) # flag_check

- asm.or(:rax, C::VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) # modified_flag

- asm.mov([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], :rax)

-

- asm.write_label(frame_flag_modified)

-

- # Push the proc on the stack

- stack_ret = ctx.stack_push(Type::Unknown)

- ep_reg = :rax

- jit_get_ep(asm, level, reg: ep_reg)

- asm.mov(:rax, [ep_reg, offs])

- asm.mov(stack_ret, :rax)

-

- KeepCompiling

- end

-

- # setblockparam

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def getblockparamproxy(jit, ctx, asm)

- # To get block_handler

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- starting_context = ctx.dup # make a copy for use with jit_chain_guard

-

- # A mirror of the interpreter code. Checking for the case

- # where it's pushing rb_block_param_proxy.

- side_exit = side_exit(jit, ctx)

-

- # EP level

- level = jit.operand(1)

-

- # Peek at the block handler so we can check whether it's nil

- comptime_handler = jit.peek_at_block_handler(level)

-

- # When a block handler is present, it should always be a GC-guarded

- # pointer (VM_BH_ISEQ_BLOCK_P)

- if comptime_handler != 0 && comptime_handler & 0x3 != 0x1

- asm.incr_counter(:getblockpp_not_gc_guarded)

- return CantCompile

- end

-

- # Load environment pointer EP from CFP

- ep_reg = :rax

- jit_get_ep(asm, level, reg: ep_reg)

-

- # Bail when VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) is non zero

- asm.test([ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS], C::VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM)

- asm.jnz(counted_exit(side_exit, :getblockpp_block_param_modified))

-

- # Load the block handler for the current frame

- # note, VM_ASSERT(VM_ENV_LOCAL_P(ep))

- block_handler = :rax

- asm.mov(block_handler, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL])

-

- # Specialize compilation for the case where no block handler is present

- if comptime_handler == 0

- # Bail if there is a block handler

- asm.cmp(block_handler, 0)

-

- jit_chain_guard(:jnz, jit, starting_context, asm, counted_exit(side_exit, :getblockpp_block_handler_none))

-

- putobject(jit, ctx, asm, val: Qnil)

- else

- # Block handler is a tagged pointer. Look at the tag. 0x03 is from VM_BH_ISEQ_BLOCK_P().

- asm.and(block_handler, 0x3)

-

- # Bail unless VM_BH_ISEQ_BLOCK_P(bh). This also checks for null.

- asm.cmp(block_handler, 0x1)

-

- jit_chain_guard(:jnz, jit, starting_context, asm, counted_exit(side_exit, :getblockpp_not_iseq_block))

-

- # Push rb_block_param_proxy. It's a root, so no need to use jit_mov_gc_ptr.

- top = ctx.stack_push(Type::BlockParamProxy)

- asm.mov(:rax, C.rb_block_param_proxy)

- asm.mov(top, :rax)

- end

-

- jump_to_next_insn(jit, ctx, asm)

-

- EndBlock

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def getspecial(jit, ctx, asm)

- # This takes two arguments, key and type

- # key is only used when type == 0

- # A non-zero type determines which type of backref to fetch

- #rb_num_t key = jit.jit_get_arg(0);

- rtype = jit.operand(1)

-

- if rtype == 0

- # not yet implemented

- return CantCompile;

- elsif rtype & 0x01 != 0

- # Fetch a "special" backref based on a char encoded by shifting by 1

-

- # Can raise if matchdata uninitialized

- jit_prepare_routine_call(jit, ctx, asm)

-

- # call rb_backref_get()

- asm.comment('rb_backref_get')

- asm.call(C.rb_backref_get)

-

- asm.mov(C_ARGS[0], C_RET) # backref

- case [rtype >> 1].pack('c')

- in ?&

- asm.comment("rb_reg_last_match")

- asm.call(C.rb_reg_last_match)

- in ?`

- asm.comment("rb_reg_match_pre")

- asm.call(C.rb_reg_match_pre)

- in ?'

- asm.comment("rb_reg_match_post")

- asm.call(C.rb_reg_match_post)

- in ?+

- asm.comment("rb_reg_match_last")

- asm.call(C.rb_reg_match_last)

- end

-

- stack_ret = ctx.stack_push(Type::Unknown)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- else

- # Fetch the N-th match from the last backref based on type shifted by 1

-

- # Can raise if matchdata uninitialized

- jit_prepare_routine_call(jit, ctx, asm)

-

- # call rb_backref_get()

- asm.comment('rb_backref_get')

- asm.call(C.rb_backref_get)

-

- # rb_reg_nth_match((int)(type >> 1), backref);

- asm.comment('rb_reg_nth_match')

- asm.mov(C_ARGS[0], rtype >> 1)

- asm.mov(C_ARGS[1], C_RET) # backref

- asm.call(C.rb_reg_nth_match)

-

- stack_ret = ctx.stack_push(Type::Unknown)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- end

- end

-

- # setspecial

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def getinstancevariable(jit, ctx, asm)

- # Specialize on a compile-time receiver, and split a block for chain guards

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- id = jit.operand(0)

- comptime_obj = jit.peek_at_self

-

- jit_getivar(jit, ctx, asm, comptime_obj, id, nil, SelfOpnd)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def setinstancevariable(jit, ctx, asm)

- starting_context = ctx.dup # make a copy for use with jit_chain_guard

-

- # Defer compilation so we can specialize on a runtime `self`

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- ivar_name = jit.operand(0)

- comptime_receiver = jit.peek_at_self

-

- # If the comptime receiver is frozen, writing an IV will raise an exception

- # and we don't want to JIT code to deal with that situation.

- if C.rb_obj_frozen_p(comptime_receiver)

- asm.incr_counter(:setivar_frozen)

- return CantCompile

- end

-

- # Check if the comptime receiver is a T_OBJECT

- receiver_t_object = C::BUILTIN_TYPE(comptime_receiver) == C::T_OBJECT

-

- # If the receiver isn't a T_OBJECT, or uses a custom allocator,

- # then just write out the IV write as a function call.

- # too-complex shapes can't use index access, so we use rb_ivar_get for them too.

- if !receiver_t_object || shape_too_complex?(comptime_receiver) || ctx.chain_depth >= 10

- asm.comment('call rb_vm_setinstancevariable')

-

- ic = jit.operand(1)

-

- # The function could raise exceptions.

- # Note that this modifies REG_SP, which is why we do it first

- jit_prepare_routine_call(jit, ctx, asm)

-

- # Get the operands from the stack

- val_opnd = ctx.stack_pop(1)

-

- # Call rb_vm_setinstancevariable(iseq, obj, id, val, ic);

- asm.mov(:rdi, jit.iseq.to_i)

- asm.mov(:rsi, [CFP, C.rb_control_frame_t.offsetof(:self)])

- asm.mov(:rdx, ivar_name)

- asm.mov(:rcx, val_opnd)

- asm.mov(:r8, ic)

- asm.call(C.rb_vm_setinstancevariable)

- else

- # Get the iv index

- shape_id = C.rb_shape_get_shape_id(comptime_receiver)

- ivar_index = C.rb_shape_get_iv_index(shape_id, ivar_name)

-

- # Get the receiver

- asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)])

-

- # Generate a side exit

- side_exit = side_exit(jit, ctx)

-

- # Upgrade type

- guard_object_is_heap(jit, ctx, asm, :rax, SelfOpnd, :setivar_not_heap)

-

- asm.comment('guard shape')

- asm.cmp(DwordPtr[:rax, C.rb_shape_id_offset], shape_id)

- megamorphic_side_exit = counted_exit(side_exit, :setivar_megamorphic)

- jit_chain_guard(:jne, jit, starting_context, asm, megamorphic_side_exit)

-

- # If we don't have an instance variable index, then we need to

- # transition out of the current shape.

- if ivar_index.nil?

- shape = C.rb_shape_get_shape_by_id(shape_id)

-

- current_capacity = shape.capacity

- dest_shape = C.rb_shape_get_next_no_warnings(shape, comptime_receiver, ivar_name)

- new_shape_id = C.rb_shape_id(dest_shape)

-

- if new_shape_id == C::OBJ_TOO_COMPLEX_SHAPE_ID

- asm.incr_counter(:setivar_too_complex)

- return CantCompile

- end

-

- ivar_index = shape.next_iv_index

-

- # If the new shape has a different capacity, we need to

- # reallocate the object.

- needs_extension = dest_shape.capacity != shape.capacity

-

- if needs_extension

- # Generate the C call so that runtime code will increase

- # the capacity and set the buffer.

- asm.mov(C_ARGS[0], :rax)

- asm.mov(C_ARGS[1], current_capacity)

- asm.mov(C_ARGS[2], dest_shape.capacity)

- asm.call(C.rb_ensure_iv_list_size)

-

- # Load the receiver again after the function call

- asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)])

- end

-

- write_val = ctx.stack_pop(1)

- jit_write_iv(asm, comptime_receiver, :rax, :rcx, ivar_index, write_val, needs_extension)

-

- # Store the new shape

- asm.comment('write shape')

- asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) # reload after jit_write_iv

- asm.mov(DwordPtr[:rax, C.rb_shape_id_offset], new_shape_id)

- else

- # If the iv index already exists, then we don't need to

- # transition to a new shape. The reason is because we find

- # the iv index by searching up the shape tree. If we've

- # made the transition already, then there's no reason to

- # update the shape on the object. Just set the IV.

- write_val = ctx.stack_pop(1)

- jit_write_iv(asm, comptime_receiver, :rax, :rcx, ivar_index, write_val, false)

- end

-

- skip_wb = asm.new_label('skip_wb')

- # If the value we're writing is an immediate, we don't need to WB

- asm.test(write_val, C::RUBY_IMMEDIATE_MASK)

- asm.jnz(skip_wb)

-

- # If the value we're writing is nil or false, we don't need to WB

- asm.cmp(write_val, Qnil)

- asm.jbe(skip_wb)

-

- asm.comment('write barrier')

- asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:self)]) # reload after jit_write_iv

- asm.mov(C_ARGS[1], write_val)

- asm.call(C.rb_gc_writebarrier)

-

- asm.write_label(skip_wb)

- end

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def getclassvariable(jit, ctx, asm)

- # rb_vm_getclassvariable can raise exceptions.

- jit_prepare_routine_call(jit, ctx, asm)

-

- asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:iseq)])

- asm.mov(C_ARGS[1], CFP)

- asm.mov(C_ARGS[2], jit.operand(0))

- asm.mov(C_ARGS[3], jit.operand(1))

- asm.call(C.rb_vm_getclassvariable)

-

- top = ctx.stack_push(Type::Unknown)

- asm.mov(top, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def setclassvariable(jit, ctx, asm)

- # rb_vm_setclassvariable can raise exceptions.

- jit_prepare_routine_call(jit, ctx, asm)

-

- asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:iseq)])

- asm.mov(C_ARGS[1], CFP)

- asm.mov(C_ARGS[2], jit.operand(0))

- asm.mov(C_ARGS[3], ctx.stack_pop(1))

- asm.mov(C_ARGS[4], jit.operand(1))

- asm.call(C.rb_vm_setclassvariable)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_getconstant_path(jit, ctx, asm)

- # Cut the block for invalidation

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- ic = C.iseq_inline_constant_cache.new(jit.operand(0))

- idlist = ic.segments

-

- # Make sure there is an exit for this block as the interpreter might want

- # to invalidate this block from rb_rjit_constant_ic_update().

- # For now, we always take an entry exit even if it was a side exit.

- Invariants.ensure_block_entry_exit(jit, cause: 'opt_getconstant_path')

-

- # See vm_ic_hit_p(). The same conditions are checked in yjit_constant_ic_update().

- ice = ic.entry

- if ice.nil?

- # In this case, leave a block that unconditionally side exits

- # for the interpreter to invalidate.

- asm.incr_counter(:optgetconst_not_cached)

- return CantCompile

- end

-

- if ice.ic_cref # with cref

- # Cache is keyed on a certain lexical scope. Use the interpreter's cache.

- side_exit = side_exit(jit, ctx)

-

- # Call function to verify the cache. It doesn't allocate or call methods.

- asm.mov(C_ARGS[0], ic.to_i)

- asm.mov(C_ARGS[1], [CFP, C.rb_control_frame_t.offsetof(:ep)])

- asm.call(C.rb_vm_ic_hit_p)

-

- # Check the result. SysV only specifies one byte for _Bool return values,

- # so it's important we only check one bit to ignore the higher bits in the register.

- asm.test(C_RET, 1)

- asm.jz(counted_exit(side_exit, :optgetconst_cache_miss))

-

- asm.mov(:rax, ic.to_i) # inline_cache

- asm.mov(:rax, [:rax, C.iseq_inline_constant_cache.offsetof(:entry)]) # ic_entry

- asm.mov(:rax, [:rax, C.iseq_inline_constant_cache_entry.offsetof(:value)]) # ic_entry_val

-

- # Push ic->entry->value

- stack_top = ctx.stack_push(Type::Unknown)

- asm.mov(stack_top, :rax)

- else # without cref

- # TODO: implement this

- # Optimize for single ractor mode.

- # if !assume_single_ractor_mode(jit, ocb)

- # return CantCompile

- # end

-

- # Invalidate output code on any constant writes associated with

- # constants referenced within the current block.

- Invariants.assume_stable_constant_names(jit, idlist)

-

- putobject(jit, ctx, asm, val: ice.value)

- end

-

- jump_to_next_insn(jit, ctx, asm)

- EndBlock

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def getconstant(jit, ctx, asm)

- id = jit.operand(0)

-

- # vm_get_ev_const can raise exceptions.

- jit_prepare_routine_call(jit, ctx, asm)

-

- allow_nil_opnd = ctx.stack_pop(1)

- klass_opnd = ctx.stack_pop(1)

-

- asm.mov(C_ARGS[0], EC)

- asm.mov(C_ARGS[1], klass_opnd)

- asm.mov(C_ARGS[2], id)

- asm.mov(C_ARGS[3], allow_nil_opnd)

- asm.call(C.rb_vm_get_ev_const)

-

- top = ctx.stack_push(Type::Unknown)

- asm.mov(top, C_RET)

-

- KeepCompiling

- end

-

- # setconstant

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def getglobal(jit, ctx, asm)

- gid = jit.operand(0)

-

- # Save the PC and SP because we might make a Ruby call for warning

- jit_prepare_routine_call(jit, ctx, asm)

-

- asm.mov(C_ARGS[0], gid)

- asm.call(C.rb_gvar_get)

-

- top = ctx.stack_push(Type::Unknown)

- asm.mov(top, C_RET)

-

- KeepCompiling

- end

-

- # setglobal

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def putnil(jit, ctx, asm)

- putobject(jit, ctx, asm, val: Qnil)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def putself(jit, ctx, asm)

- stack_top = ctx.stack_push_self

- asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)])

- asm.mov(stack_top, :rax)

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def putobject(jit, ctx, asm, val: jit.operand(0))

- # Push it to the stack

- val_type = Type.from(C.to_ruby(val))

- stack_top = ctx.stack_push(val_type)

- if asm.imm32?(val)

- asm.mov(stack_top, val)

- else # 64-bit immediates can't be directly written to memory

- asm.mov(:rax, val)

- asm.mov(stack_top, :rax)

- end

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def putspecialobject(jit, ctx, asm)

- object_type = jit.operand(0)

- if object_type == C::VM_SPECIAL_OBJECT_VMCORE

- stack_top = ctx.stack_push(Type::UnknownHeap)

- asm.mov(:rax, C.rb_mRubyVMFrozenCore)

- asm.mov(stack_top, :rax)

- KeepCompiling

- else

- # TODO: implement for VM_SPECIAL_OBJECT_CBASE and

- # VM_SPECIAL_OBJECT_CONST_BASE

- CantCompile

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def putstring(jit, ctx, asm)

- put_val = jit.operand(0, ruby: true)

-

- # Save the PC and SP because the callee will allocate

- jit_prepare_routine_call(jit, ctx, asm)

-

- asm.mov(C_ARGS[0], EC)

- asm.mov(C_ARGS[1], to_value(put_val))

- asm.mov(C_ARGS[2], 0)

- asm.call(C.rb_ec_str_resurrect)

-

- stack_top = ctx.stack_push(Type::TString)

- asm.mov(stack_top, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def putchilledstring(jit, ctx, asm)

- put_val = jit.operand(0, ruby: true)

-

- # Save the PC and SP because the callee will allocate

- jit_prepare_routine_call(jit, ctx, asm)

-

- asm.mov(C_ARGS[0], EC)

- asm.mov(C_ARGS[1], to_value(put_val))

- asm.mov(C_ARGS[2], 1)

- asm.call(C.rb_ec_str_resurrect)

-

- stack_top = ctx.stack_push(Type::TString)

- asm.mov(stack_top, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def concatstrings(jit, ctx, asm)

- n = jit.operand(0)

-

- # Save the PC and SP because we are allocating

- jit_prepare_routine_call(jit, ctx, asm)

-

- asm.lea(:rax, ctx.sp_opnd(-C.VALUE.size * n))

-

- # call rb_str_concat_literals(size_t n, const VALUE *strings);

- asm.mov(C_ARGS[0], n)

- asm.mov(C_ARGS[1], :rax)

- asm.call(C.rb_str_concat_literals)

-

- ctx.stack_pop(n)

- stack_ret = ctx.stack_push(Type::TString)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def anytostring(jit, ctx, asm)

- # Save the PC and SP since we might call #to_s

- jit_prepare_routine_call(jit, ctx, asm)

-

- str = ctx.stack_pop(1)

- val = ctx.stack_pop(1)

-

- asm.mov(C_ARGS[0], str)

- asm.mov(C_ARGS[1], val)

- asm.call(C.rb_obj_as_string_result)

-

- # Push the return value

- stack_ret = ctx.stack_push(Type::TString)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def toregexp(jit, ctx, asm)

- opt = jit.operand(0, signed: true)

- cnt = jit.operand(1)

-

- # Save the PC and SP because this allocates an object and could

- # raise an exception.

- jit_prepare_routine_call(jit, ctx, asm)

-

- asm.lea(:rax, ctx.sp_opnd(-C.VALUE.size * cnt)) # values_ptr

- ctx.stack_pop(cnt)

-

- asm.mov(C_ARGS[0], 0)

- asm.mov(C_ARGS[1], cnt)

- asm.mov(C_ARGS[2], :rax) # values_ptr

- asm.call(C.rb_ary_tmp_new_from_values)

-

- # Save the array so we can clear it later

- asm.push(C_RET)

- asm.push(C_RET) # Alignment

-

- asm.mov(C_ARGS[0], C_RET)

- asm.mov(C_ARGS[1], opt)

- asm.call(C.rb_reg_new_ary)

-

- # The actual regex is in RAX now. Pop the temp array from

- # rb_ary_tmp_new_from_values into C arg regs so we can clear it

- asm.pop(:rcx) # Alignment

- asm.pop(:rcx) # ary

-

- # The value we want to push on the stack is in RAX right now

- stack_ret = ctx.stack_push(Type::UnknownHeap)

- asm.mov(stack_ret, C_RET)

-

- # Clear the temp array.

- asm.mov(C_ARGS[0], :rcx) # ary

- asm.call(C.rb_ary_clear)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def intern(jit, ctx, asm)

- # Save the PC and SP because we might allocate

- jit_prepare_routine_call(jit, ctx, asm);

-

- str = ctx.stack_pop(1)

- asm.mov(C_ARGS[0], str)

- asm.call(C.rb_str_intern)

-

- # Push the return value

- stack_ret = ctx.stack_push(Type::Unknown)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def newarray(jit, ctx, asm)

- n = jit.operand(0)

-

- # Save the PC and SP because we are allocating

- jit_prepare_routine_call(jit, ctx, asm)

-

- # If n is 0, then elts is never going to be read, so we can just pass null

- if n == 0

- values_ptr = 0

- else

- asm.comment('load pointer to array elts')

- offset_magnitude = C.VALUE.size * n

- values_opnd = ctx.sp_opnd(-(offset_magnitude))

- asm.lea(:rax, values_opnd)

- values_ptr = :rax

- end

-

- # call rb_ec_ary_new_from_values(struct rb_execution_context_struct *ec, long n, const VALUE *elts);

- asm.mov(C_ARGS[0], EC)

- asm.mov(C_ARGS[1], n)

- asm.mov(C_ARGS[2], values_ptr)

- asm.call(C.rb_ec_ary_new_from_values)

-

- ctx.stack_pop(n)

- stack_ret = ctx.stack_push(Type::TArray)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def duparray(jit, ctx, asm)

- ary = jit.operand(0)

-

- # Save the PC and SP because we are allocating

- jit_prepare_routine_call(jit, ctx, asm)

-

- # call rb_ary_resurrect(VALUE ary);

- asm.comment('call rb_ary_resurrect')

- asm.mov(C_ARGS[0], ary)

- asm.call(C.rb_ary_resurrect)

-

- stack_ret = ctx.stack_push(Type::TArray)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- end

-

- # duphash

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def expandarray(jit, ctx, asm)

- # Both arguments are rb_num_t which is unsigned

- num = jit.operand(0)

- flag = jit.operand(1)

-

- # If this instruction has the splat flag, then bail out.

- if flag & 0x01 != 0

- asm.incr_counter(:expandarray_splat)

- return CantCompile

- end

-

- # If this instruction has the postarg flag, then bail out.

- if flag & 0x02 != 0

- asm.incr_counter(:expandarray_postarg)

- return CantCompile

- end

-

- side_exit = side_exit(jit, ctx)

-

- array_opnd = ctx.stack_opnd(0)

- array_stack_opnd = StackOpnd[0]

-

- # num is the number of requested values. If there aren't enough in the

- # array then we're going to push on nils.

- if ctx.get_opnd_type(array_stack_opnd) == Type::Nil

- ctx.stack_pop(1) # pop after using the type info

- # special case for a, b = nil pattern

- # push N nils onto the stack

- num.times do

- push_opnd = ctx.stack_push(Type::Nil)

- asm.mov(push_opnd, Qnil)

- end

- return KeepCompiling

- end

-

- # Move the array from the stack and check that it's an array.

- asm.mov(:rax, array_opnd)

- guard_object_is_array(jit, ctx, asm, :rax, :rcx, array_stack_opnd, :expandarray_not_array)

- ctx.stack_pop(1) # pop after using the type info

-

- # If we don't actually want any values, then just return.

- if num == 0

- return KeepCompiling

- end

-

- jit_array_len(asm, :rax, :rcx)

-

- # Only handle the case where the number of values in the array is greater

- # than or equal to the number of values requested.

- asm.cmp(:rcx, num)

- asm.jl(counted_exit(side_exit, :expandarray_rhs_too_small))

-

- # Conditionally load the address of the heap array into REG1.

- # (struct RArray *)(obj)->as.heap.ptr

- #asm.mov(:rax, array_opnd)

- asm.mov(:rcx, [:rax, C.RBasic.offsetof(:flags)])

- asm.test(:rcx, C::RARRAY_EMBED_FLAG);

- asm.mov(:rcx, [:rax, C.RArray.offsetof(:as, :heap, :ptr)])

-

- # Load the address of the embedded array into REG1.

- # (struct RArray *)(obj)->as.ary

- asm.lea(:rax, [:rax, C.RArray.offsetof(:as, :ary)])

-

- asm.cmovnz(:rcx, :rax)

-

- # Loop backward through the array and push each element onto the stack.

- (num - 1).downto(0).each do |i|

- top = ctx.stack_push(Type::Unknown)

- asm.mov(:rax, [:rcx, i * C.VALUE.size])

- asm.mov(top, :rax)

- end

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def concatarray(jit, ctx, asm)

- # Save the PC and SP because the callee may allocate

- # Note that this modifies REG_SP, which is why we do it first

- jit_prepare_routine_call(jit, ctx, asm)

-

- # Get the operands from the stack

- ary2st_opnd = ctx.stack_pop(1)

- ary1_opnd = ctx.stack_pop(1)

-

- # Call rb_vm_concat_array(ary1, ary2st)

- asm.mov(C_ARGS[0], ary1_opnd)

- asm.mov(C_ARGS[1], ary2st_opnd)

- asm.call(C.rb_vm_concat_array)

-

- stack_ret = ctx.stack_push(Type::TArray)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def splatarray(jit, ctx, asm)

- flag = jit.operand(0)

-

- # Save the PC and SP because the callee may allocate

- # Note that this modifies REG_SP, which is why we do it first

- jit_prepare_routine_call(jit, ctx, asm)

-

- # Get the operands from the stack

- ary_opnd = ctx.stack_pop(1)

-

- # Call rb_vm_splat_array(flag, ary)

- asm.mov(C_ARGS[0], flag)

- asm.mov(C_ARGS[1], ary_opnd)

- asm.call(C.rb_vm_splat_array)

-

- stack_ret = ctx.stack_push(Type::TArray)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def newhash(jit, ctx, asm)

- num = jit.operand(0)

-

- # Save the PC and SP because we are allocating

- jit_prepare_routine_call(jit, ctx, asm)

-

- if num != 0

- # val = rb_hash_new_with_size(num / 2);

- asm.mov(C_ARGS[0], num / 2)

- asm.call(C.rb_hash_new_with_size)

-

- # Save the allocated hash as we want to push it after insertion

- asm.push(C_RET)

- asm.push(C_RET) # x86 alignment

-

- # Get a pointer to the values to insert into the hash

- asm.lea(:rcx, ctx.stack_opnd(num - 1))

-

- # rb_hash_bulk_insert(num, STACK_ADDR_FROM_TOP(num), val);

- asm.mov(C_ARGS[0], num)

- asm.mov(C_ARGS[1], :rcx)

- asm.mov(C_ARGS[2], C_RET)

- asm.call(C.rb_hash_bulk_insert)

-

- asm.pop(:rax)

- asm.pop(:rax)

-

- ctx.stack_pop(num)

- stack_ret = ctx.stack_push(Type::Hash)

- asm.mov(stack_ret, :rax)

- else

- # val = rb_hash_new();

- asm.call(C.rb_hash_new)

- stack_ret = ctx.stack_push(Type::Hash)

- asm.mov(stack_ret, C_RET)

- end

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def newrange(jit, ctx, asm)

- flag = jit.operand(0)

-

- # rb_range_new() allocates and can raise

- jit_prepare_routine_call(jit, ctx, asm)

-

- # val = rb_range_new(low, high, (int)flag);

- asm.mov(C_ARGS[0], ctx.stack_opnd(1))

- asm.mov(C_ARGS[1], ctx.stack_opnd(0))

- asm.mov(C_ARGS[2], flag)

- asm.call(C.rb_range_new)

-

- ctx.stack_pop(2)

- stack_ret = ctx.stack_push(Type::UnknownHeap)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def pop(jit, ctx, asm)

- ctx.stack_pop

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def dup(jit, ctx, asm)

- dup_val = ctx.stack_opnd(0)

- mapping, tmp_type = ctx.get_opnd_mapping(StackOpnd[0])

-

- loc0 = ctx.stack_push_mapping([mapping, tmp_type])

- asm.mov(:rax, dup_val)

- asm.mov(loc0, :rax)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def dupn(jit, ctx, asm)

- n = jit.operand(0)

-

- # In practice, seems to be only used for n==2

- if n != 2

- return CantCompile

- end

-

- opnd1 = ctx.stack_opnd(1)

- opnd0 = ctx.stack_opnd(0)

-

- mapping1 = ctx.get_opnd_mapping(StackOpnd[1])

- mapping0 = ctx.get_opnd_mapping(StackOpnd[0])

-

- dst1 = ctx.stack_push_mapping(mapping1)

- asm.mov(:rax, opnd1)

- asm.mov(dst1, :rax)

-

- dst0 = ctx.stack_push_mapping(mapping0)

- asm.mov(:rax, opnd0)

- asm.mov(dst0, :rax)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def swap(jit, ctx, asm)

- stack_swap(jit, ctx, asm, 0, 1)

- KeepCompiling

- end

-

- # opt_reverse

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def topn(jit, ctx, asm)

- n = jit.operand(0)

-

- top_n_val = ctx.stack_opnd(n)

- mapping = ctx.get_opnd_mapping(StackOpnd[n])

- loc0 = ctx.stack_push_mapping(mapping)

- asm.mov(:rax, top_n_val)

- asm.mov(loc0, :rax)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def setn(jit, ctx, asm)

- n = jit.operand(0)

-

- top_val = ctx.stack_pop(0)

- dst_opnd = ctx.stack_opnd(n)

- asm.mov(:rax, top_val)

- asm.mov(dst_opnd, :rax)

-

- mapping = ctx.get_opnd_mapping(StackOpnd[0])

- ctx.set_opnd_mapping(StackOpnd[n], mapping)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def adjuststack(jit, ctx, asm)

- n = jit.operand(0)

- ctx.stack_pop(n)

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def defined(jit, ctx, asm)

- op_type = jit.operand(0)

- obj = jit.operand(1, ruby: true)

- pushval = jit.operand(2, ruby: true)

-

- # Save the PC and SP because the callee may allocate

- # Note that this modifies REG_SP, which is why we do it first

- jit_prepare_routine_call(jit, ctx, asm)

-

- # Get the operands from the stack

- v_opnd = ctx.stack_pop(1)

-

- # Call vm_defined(ec, reg_cfp, op_type, obj, v)

- asm.mov(C_ARGS[0], EC)

- asm.mov(C_ARGS[1], CFP)

- asm.mov(C_ARGS[2], op_type)

- asm.mov(C_ARGS[3], to_value(obj))

- asm.mov(C_ARGS[4], v_opnd)

- asm.call(C.rb_vm_defined)

-

- asm.test(C_RET, 255)

- asm.mov(:rax, Qnil)

- asm.mov(:rcx, to_value(pushval))

- asm.cmovnz(:rax, :rcx)

-

- # Push the return value onto the stack

- out_type = if C::SPECIAL_CONST_P(pushval)

- Type::UnknownImm

- else

- Type::Unknown

- end

- stack_ret = ctx.stack_push(out_type)

- asm.mov(stack_ret, :rax)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def definedivar(jit, ctx, asm)

- # Defer compilation so we can specialize base on a runtime receiver

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- ivar_name = jit.operand(0)

- # Value that will be pushed on the stack if the ivar is defined. In practice this is always the

- # string "instance-variable". If the ivar is not defined, nil will be pushed instead.

- pushval = jit.operand(2, ruby: true)

-

- # Get the receiver

- recv = :rcx

- asm.mov(recv, [CFP, C.rb_control_frame_t.offsetof(:self)])

-

- # Specialize base on compile time values

- comptime_receiver = jit.peek_at_self

-

- if shape_too_complex?(comptime_receiver)

- # Fall back to calling rb_ivar_defined

-

- # Save the PC and SP because the callee may allocate

- # Note that this modifies REG_SP, which is why we do it first

- jit_prepare_routine_call(jit, ctx, asm) # clobbers :rax

-

- # Call rb_ivar_defined(recv, ivar_name)

- asm.mov(C_ARGS[0], recv)

- asm.mov(C_ARGS[1], ivar_name)

- asm.call(C.rb_ivar_defined)

-

- # if (rb_ivar_defined(recv, ivar_name)) {

- # val = pushval;

- # }

- asm.test(C_RET, 255)

- asm.mov(:rax, Qnil)

- asm.mov(:rcx, to_value(pushval))

- asm.cmovnz(:rax, :rcx)

-

- # Push the return value onto the stack

- out_type = C::SPECIAL_CONST_P(pushval) ? Type::UnknownImm : Type::Unknown

- stack_ret = ctx.stack_push(out_type)

- asm.mov(stack_ret, :rax)

-

- return KeepCompiling

- end

-

- shape_id = C.rb_shape_get_shape_id(comptime_receiver)

- ivar_exists = C.rb_shape_get_iv_index(shape_id, ivar_name)

-

- side_exit = side_exit(jit, ctx)

-

- # Guard heap object (recv_opnd must be used before stack_pop)

- guard_object_is_heap(jit, ctx, asm, recv, SelfOpnd)

-

- shape_opnd = DwordPtr[recv, C.rb_shape_id_offset]

-

- asm.comment('guard shape')

- asm.cmp(shape_opnd, shape_id)

- jit_chain_guard(:jne, jit, ctx, asm, side_exit)

-

- result = ivar_exists ? C.to_value(pushval) : Qnil

- putobject(jit, ctx, asm, val: result)

-

- # Jump to next instruction. This allows guard chains to share the same successor.

- jump_to_next_insn(jit, ctx, asm)

-

- return EndBlock

- end

-

- # checkmatch

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def checkkeyword(jit, ctx, asm)

- # When a keyword is unspecified past index 32, a hash will be used

- # instead. This can only happen in iseqs taking more than 32 keywords.

- if jit.iseq.body.param.keyword.num >= 32

- return CantCompile

- end

-

- # The EP offset to the undefined bits local

- bits_offset = jit.operand(0)

-

- # The index of the keyword we want to check

- index = jit.operand(1, signed: true)

-

- # Load environment pointer EP

- ep_reg = :rax

- jit_get_ep(asm, 0, reg: ep_reg)

-

- # VALUE kw_bits = *(ep - bits)

- bits_opnd = [ep_reg, C.VALUE.size * -bits_offset]

-

- # unsigned int b = (unsigned int)FIX2ULONG(kw_bits);

- # if ((b & (0x01 << idx))) {

- #

- # We can skip the FIX2ULONG conversion by shifting the bit we test

- bit_test = 0x01 << (index + 1)

- asm.test(bits_opnd, bit_test)

- asm.mov(:rax, Qfalse)

- asm.mov(:rcx, Qtrue)

- asm.cmovz(:rax, :rcx)

-

- stack_ret = ctx.stack_push(Type::UnknownImm)

- asm.mov(stack_ret, :rax)

-

- KeepCompiling

- end

-

- # checktype

- # defineclass

- # definemethod

- # definesmethod

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def send(jit, ctx, asm)

- # Specialize on a compile-time receiver, and split a block for chain guards

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- cd = C.rb_call_data.new(jit.operand(0))

- blockiseq = jit.operand(1)

-

- # calling->ci

- mid = C.vm_ci_mid(cd.ci)

- calling = build_calling(ci: cd.ci, block_handler: blockiseq)

-

- if calling.flags & C::VM_CALL_FORWARDING != 0

- return CantCompile

- end

-

- # vm_sendish

- cme, comptime_recv_klass = jit_search_method(jit, ctx, asm, mid, calling)

- if cme == CantCompile

- return CantCompile

- end

- jit_call_general(jit, ctx, asm, mid, calling, cme, comptime_recv_klass)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_send_without_block(jit, ctx, asm, cd: C.rb_call_data.new(jit.operand(0)))

- # Specialize on a compile-time receiver, and split a block for chain guards

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- # calling->ci

- mid = C.vm_ci_mid(cd.ci)

- calling = build_calling(ci: cd.ci, block_handler: C::VM_BLOCK_HANDLER_NONE)

-

- # vm_sendish

- cme, comptime_recv_klass = jit_search_method(jit, ctx, asm, mid, calling)

- if cme == CantCompile

- return CantCompile

- end

- jit_call_general(jit, ctx, asm, mid, calling, cme, comptime_recv_klass)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def objtostring(jit, ctx, asm)

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- recv = ctx.stack_opnd(0)

- comptime_recv = jit.peek_at_stack(0)

-

- if C.RB_TYPE_P(comptime_recv, C::RUBY_T_STRING)

- side_exit = side_exit(jit, ctx)

-

- jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv, StackOpnd[0], comptime_recv, side_exit)

- # No work needed. The string value is already on the top of the stack.

- KeepCompiling

- else

- cd = C.rb_call_data.new(jit.operand(0))

- opt_send_without_block(jit, ctx, asm, cd:)

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_ary_freeze(jit, ctx, asm)

- unless Invariants.assume_bop_not_redefined(jit, C::ARRAY_REDEFINED_OP_FLAG, C::BOP_FREEZE)

- return CantCompile;

- end

-

- ary = jit.operand(0, ruby: true)

-

- # Push the return value onto the stack

- stack_ret = ctx.stack_push(Type::CArray)

- asm.mov(:rax, to_value(ary))

- asm.mov(stack_ret, :rax)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_hash_freeze(jit, ctx, asm)

- unless Invariants.assume_bop_not_redefined(jit, C::HASH_REDEFINED_OP_FLAG, C::BOP_FREEZE)

- return CantCompile;

- end

-

- hash = jit.operand(0, ruby: true)

-

- # Push the return value onto the stack

- stack_ret = ctx.stack_push(Type::CHash)

- asm.mov(:rax, to_value(hash))

- asm.mov(stack_ret, :rax)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_str_freeze(jit, ctx, asm)

- unless Invariants.assume_bop_not_redefined(jit, C::STRING_REDEFINED_OP_FLAG, C::BOP_FREEZE)

- return CantCompile;

- end

-

- str = jit.operand(0, ruby: true)

-

- # Push the return value onto the stack

- stack_ret = ctx.stack_push(Type::CString)

- asm.mov(:rax, to_value(str))

- asm.mov(stack_ret, :rax)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_nil_p(jit, ctx, asm)

- opt_send_without_block(jit, ctx, asm)

- end

-

- # opt_str_uminus

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_newarray_send(jit, ctx, asm)

- type = C.ID2SYM jit.operand(1)

-

- case type

- when :min then opt_newarray_min(jit, ctx, asm)

- when :max then opt_newarray_max(jit, ctx, asm)

- when :hash then opt_newarray_hash(jit, ctx, asm)

- else

- return CantCompile

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_newarray_min(jit, ctx, asm)

- num = jit.operand(0)

-

- # Save the PC and SP because we may allocate

- jit_prepare_routine_call(jit, ctx, asm)

-

- offset_magnitude = C.VALUE.size * num

- values_opnd = ctx.sp_opnd(-offset_magnitude)

- asm.lea(:rax, values_opnd)

-

- asm.mov(C_ARGS[0], EC)

- asm.mov(C_ARGS[1], num)

- asm.mov(C_ARGS[2], :rax)

- asm.call(C.rb_vm_opt_newarray_min)

-

- ctx.stack_pop(num)

- stack_ret = ctx.stack_push(Type::Unknown)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_newarray_max(jit, ctx, asm)

- num = jit.operand(0)

-

- # Save the PC and SP because we may allocate

- jit_prepare_routine_call(jit, ctx, asm)

-

- offset_magnitude = C.VALUE.size * num

- values_opnd = ctx.sp_opnd(-offset_magnitude)

- asm.lea(:rax, values_opnd)

-

- asm.mov(C_ARGS[0], EC)

- asm.mov(C_ARGS[1], num)

- asm.mov(C_ARGS[2], :rax)

- asm.call(C.rb_vm_opt_newarray_max)

-

- ctx.stack_pop(num)

- stack_ret = ctx.stack_push(Type::Unknown)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_newarray_hash(jit, ctx, asm)

- num = jit.operand(0)

-

- # Save the PC and SP because we may allocate

- jit_prepare_routine_call(jit, ctx, asm)

-

- offset_magnitude = C.VALUE.size * num

- values_opnd = ctx.sp_opnd(-offset_magnitude)

- asm.lea(:rax, values_opnd)

-

- asm.mov(C_ARGS[0], EC)

- asm.mov(C_ARGS[1], num)

- asm.mov(C_ARGS[2], :rax)

- asm.call(C.rb_vm_opt_newarray_hash)

-

- ctx.stack_pop(num)

- stack_ret = ctx.stack_push(Type::Unknown)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def invokesuper(jit, ctx, asm)

- cd = C.rb_call_data.new(jit.operand(0))

- block = jit.operand(1)

-

- # Defer compilation so we can specialize on class of receiver

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- me = C.rb_vm_frame_method_entry(jit.cfp)

- if me.nil?

- return CantCompile

- end

-

- # FIXME: We should track and invalidate this block when this cme is invalidated

- current_defined_class = me.defined_class

- mid = me.def.original_id

-

- if me.to_i != C.rb_callable_method_entry(current_defined_class, me.called_id).to_i

- # Though we likely could generate this call, as we are only concerned

- # with the method entry remaining valid, assume_method_lookup_stable

- # below requires that the method lookup matches as well

- return CantCompile

- end

-

- # vm_search_normal_superclass

- rbasic_klass = C.to_ruby(C.RBasic.new(C.to_value(current_defined_class)).klass)

- if C::BUILTIN_TYPE(current_defined_class) == C::RUBY_T_ICLASS && C::BUILTIN_TYPE(rbasic_klass) == C::RUBY_T_MODULE && \

- C::FL_TEST_RAW(rbasic_klass, C::RMODULE_IS_REFINEMENT)

- return CantCompile

- end

- comptime_superclass = C.rb_class_get_superclass(C.RCLASS_ORIGIN(current_defined_class))

-

- ci = cd.ci

- argc = C.vm_ci_argc(ci)

-

- ci_flags = C.vm_ci_flag(ci)

-

- # Don't JIT calls that aren't simple

- # Note, not using VM_CALL_ARGS_SIMPLE because sometimes we pass a block.

-

- if ci_flags & C::VM_CALL_KWARG != 0

- asm.incr_counter(:send_keywords)

- return CantCompile

- end

- if ci_flags & C::VM_CALL_KW_SPLAT != 0

- asm.incr_counter(:send_kw_splat)

- return CantCompile

- end

- if ci_flags & C::VM_CALL_ARGS_BLOCKARG != 0

- asm.incr_counter(:send_block_arg)

- return CantCompile

- end

-

- # Ensure we haven't rebound this method onto an incompatible class.

- # In the interpreter we try to avoid making this check by performing some

- # cheaper calculations first, but since we specialize on the method entry

- # and so only have to do this once at compile time this is fine to always

- # check and side exit.

- comptime_recv = jit.peek_at_stack(argc)

- unless C.obj_is_kind_of(comptime_recv, current_defined_class)

- return CantCompile

- end

-

- # Do method lookup

- cme = C.rb_callable_method_entry(comptime_superclass, mid)

-

- if cme.nil?

- return CantCompile

- end

-

- # Check that we'll be able to write this method dispatch before generating checks

- cme_def_type = cme.def.type

- if cme_def_type != C::VM_METHOD_TYPE_ISEQ && cme_def_type != C::VM_METHOD_TYPE_CFUNC

- # others unimplemented

- return CantCompile

- end

-

- asm.comment('guard known me')

- lep_opnd = :rax

- jit_get_lep(jit, asm, reg: lep_opnd)

- ep_me_opnd = [lep_opnd, C.VALUE.size * C::VM_ENV_DATA_INDEX_ME_CREF]

-

- asm.mov(:rcx, me.to_i)

- asm.cmp(ep_me_opnd, :rcx)

- asm.jne(counted_exit(side_exit(jit, ctx), :invokesuper_me_changed))

-

- if block == C::VM_BLOCK_HANDLER_NONE

- # Guard no block passed

- # rb_vm_frame_block_handler(GET_EC()->cfp) == VM_BLOCK_HANDLER_NONE

- # note, we assume VM_ASSERT(VM_ENV_LOCAL_P(ep))

- #

- # TODO: this could properly forward the current block handler, but

- # would require changes to gen_send_*

- asm.comment('guard no block given')

- ep_specval_opnd = [lep_opnd, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]

- asm.cmp(ep_specval_opnd, C::VM_BLOCK_HANDLER_NONE)

- asm.jne(counted_exit(side_exit(jit, ctx), :invokesuper_block))

- end

-

- # We need to assume that both our current method entry and the super

- # method entry we invoke remain stable

- Invariants.assume_method_lookup_stable(jit, me)

- Invariants.assume_method_lookup_stable(jit, cme)

-

- # Method calls may corrupt types

- ctx.clear_local_types

-

- calling = build_calling(ci:, block_handler: block)

- case cme_def_type

- in C::VM_METHOD_TYPE_ISEQ

- iseq = def_iseq_ptr(cme.def)

- frame_type = C::VM_FRAME_MAGIC_METHOD | C::VM_ENV_FLAG_LOCAL

- jit_call_iseq(jit, ctx, asm, cme, calling, iseq, frame_type:)

- in C::VM_METHOD_TYPE_CFUNC

- jit_call_cfunc(jit, ctx, asm, cme, calling)

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def invokeblock(jit, ctx, asm)

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- # Get call info

- cd = C.rb_call_data.new(jit.operand(0))

- calling = build_calling(ci: cd.ci, block_handler: :captured)

-

- # Get block_handler

- cfp = jit.cfp

- lep = C.rb_vm_ep_local_ep(cfp.ep)

- comptime_handler = lep[C::VM_ENV_DATA_INDEX_SPECVAL]

-

- # Handle each block_handler type

- if comptime_handler == C::VM_BLOCK_HANDLER_NONE # no block given

- asm.incr_counter(:invokeblock_none)

- CantCompile

- elsif comptime_handler & 0x3 == 0x1 # VM_BH_ISEQ_BLOCK_P

- asm.comment('get local EP')

- ep_reg = :rax

- jit_get_lep(jit, asm, reg: ep_reg)

- asm.mov(:rax, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler_opnd

-

- asm.comment('guard block_handler type')

- side_exit = side_exit(jit, ctx)

- asm.mov(:rcx, :rax)

- asm.and(:rcx, 0x3) # block_handler is a tagged pointer

- asm.cmp(:rcx, 0x1) # VM_BH_ISEQ_BLOCK_P

- tag_changed_exit = counted_exit(side_exit, :invokeblock_tag_changed)

- jit_chain_guard(:jne, jit, ctx, asm, tag_changed_exit)

-

- comptime_captured = C.rb_captured_block.new(comptime_handler & ~0x3)

- comptime_iseq = comptime_captured.code.iseq

-

- asm.comment('guard known ISEQ')

- asm.and(:rax, ~0x3) # captured

- asm.mov(:rax, [:rax, C.VALUE.size * 2]) # captured->iseq

- asm.mov(:rcx, comptime_iseq.to_i)

- asm.cmp(:rax, :rcx)

- block_changed_exit = counted_exit(side_exit, :invokeblock_iseq_block_changed)

- jit_chain_guard(:jne, jit, ctx, asm, block_changed_exit)

-

- jit_call_iseq(jit, ctx, asm, nil, calling, comptime_iseq, frame_type: C::VM_FRAME_MAGIC_BLOCK)

- elsif comptime_handler & 0x3 == 0x3 # VM_BH_IFUNC_P

- # We aren't handling CALLER_SETUP_ARG and CALLER_REMOVE_EMPTY_KW_SPLAT yet.

- if calling.flags & C::VM_CALL_ARGS_SPLAT != 0

- asm.incr_counter(:invokeblock_ifunc_args_splat)

- return CantCompile

- end

- if calling.flags & C::VM_CALL_KW_SPLAT != 0

- asm.incr_counter(:invokeblock_ifunc_kw_splat)

- return CantCompile

- end

-

- asm.comment('get local EP')

- jit_get_lep(jit, asm, reg: :rax)

- asm.mov(:rcx, [:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler_opnd

-

- asm.comment('guard block_handler type');

- side_exit = side_exit(jit, ctx)

- asm.mov(:rax, :rcx) # block_handler_opnd

- asm.and(:rax, 0x3) # tag_opnd: block_handler is a tagged pointer

- asm.cmp(:rax, 0x3) # VM_BH_IFUNC_P

- tag_changed_exit = counted_exit(side_exit, :invokeblock_tag_changed)

- jit_chain_guard(:jne, jit, ctx, asm, tag_changed_exit)

-

- # The cfunc may not be leaf

- jit_prepare_routine_call(jit, ctx, asm) # clobbers :rax

-

- asm.comment('call ifunc')

- asm.and(:rcx, ~0x3) # captured_opnd

- asm.lea(:rax, ctx.sp_opnd(-calling.argc * C.VALUE.size)) # argv

- asm.mov(C_ARGS[0], EC)

- asm.mov(C_ARGS[1], :rcx) # captured_opnd

- asm.mov(C_ARGS[2], calling.argc)

- asm.mov(C_ARGS[3], :rax) # argv

- asm.call(C.rb_vm_yield_with_cfunc)

-

- ctx.stack_pop(calling.argc)

- stack_ret = ctx.stack_push(Type::Unknown)

- asm.mov(stack_ret, C_RET)

-

- # cfunc calls may corrupt types

- ctx.clear_local_types

-

- # Share the successor with other chains

- jump_to_next_insn(jit, ctx, asm)

- EndBlock

- elsif symbol?(comptime_handler)

- asm.incr_counter(:invokeblock_symbol)

- CantCompile

- else # Proc

- asm.incr_counter(:invokeblock_proc)

- CantCompile

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def leave(jit, ctx, asm)

- assert_equal(ctx.stack_size, 1)

-

- jit_check_ints(jit, ctx, asm)

-

- asm.comment('pop stack frame')

- asm.lea(:rax, [CFP, C.rb_control_frame_t.size])

- asm.mov(CFP, :rax)

- asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], :rax)

-

- # Return a value (for compile_leave_exit)

- ret_opnd = ctx.stack_pop

- asm.mov(:rax, ret_opnd)

-

- # Set caller's SP and push a value to its stack (for JIT)

- asm.mov(SP, [CFP, C.rb_control_frame_t.offsetof(:sp)]) # Note: SP is in the position after popping a receiver and arguments

- asm.mov([SP], :rax)

-

- # Jump to cfp->jit_return

- asm.jmp([CFP, -C.rb_control_frame_t.size + C.rb_control_frame_t.offsetof(:jit_return)])

-

- EndBlock

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def throw(jit, ctx, asm)

- throw_state = jit.operand(0)

- asm.mov(:rcx, ctx.stack_pop(1)) # throwobj

-

- # THROW_DATA_NEW allocates. Save SP for GC and PC for allocation tracing as

- # well as handling the catch table. However, not using jit_prepare_routine_call

- # since we don't need a patch point for this implementation.

- jit_save_pc(jit, asm) # clobbers rax

- jit_save_sp(ctx, asm)

-

- # rb_vm_throw verifies it's a valid throw, sets ec->tag->state, and returns throw

- # data, which is throwobj or a vm_throw_data wrapping it. When ec->tag->state is

- # set, JIT code callers will handle the throw with vm_exec_handle_exception.

- asm.mov(C_ARGS[0], EC)

- asm.mov(C_ARGS[1], CFP)

- asm.mov(C_ARGS[2], throw_state)

- # asm.mov(C_ARGS[3], :rcx) # same reg

- asm.call(C.rb_vm_throw)

-

- asm.comment('exit from throw')

- asm.pop(SP)

- asm.pop(EC)

- asm.pop(CFP)

-

- # return C_RET as C_RET

- asm.ret

- EndBlock

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jump(jit, ctx, asm)

- # Check for interrupts, but only on backward branches that may create loops

- jump_offset = jit.operand(0, signed: true)

- if jump_offset < 0

- jit_check_ints(jit, ctx, asm)

- end

-

- pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset)

- jit_direct_jump(jit.iseq, pc, ctx, asm)

- EndBlock

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def branchif(jit, ctx, asm)

- # Check for interrupts, but only on backward branches that may create loops

- jump_offset = jit.operand(0, signed: true)

- if jump_offset < 0

- jit_check_ints(jit, ctx, asm)

- end

-

- # Get the branch target instruction offsets

- next_pc = jit.pc + C.VALUE.size * jit.insn.len

- jump_pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset)

-

- val_type = ctx.get_opnd_type(StackOpnd[0])

- val_opnd = ctx.stack_pop(1)

-

- if (result = val_type.known_truthy) != nil

- target_pc = result ? jump_pc : next_pc

- jit_direct_jump(jit.iseq, target_pc, ctx, asm)

- else

- # This `test` sets ZF only for Qnil and Qfalse, which let jz jump.

- asm.test(val_opnd, ~Qnil)

-

- # Set stubs

- branch_stub = BranchStub.new(

- iseq: jit.iseq,

- shape: Default,

- target0: BranchTarget.new(ctx:, pc: jump_pc), # branch target

- target1: BranchTarget.new(ctx:, pc: next_pc), # fallthrough

- )

- branch_stub.target0.address = Assembler.new.then do |ocb_asm|

- @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true)

- @ocb.write(ocb_asm)

- end

- branch_stub.target1.address = Assembler.new.then do |ocb_asm|

- @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false)

- @ocb.write(ocb_asm)

- end

-

- # Jump to target0 on jnz

- branch_stub.compile = compile_branchif(branch_stub)

- branch_stub.compile.call(asm)

- end

-

- EndBlock

- end

-

- def compile_branchif(branch_stub) # Proc escapes arguments in memory

- proc do |branch_asm|

- branch_asm.comment("branchif #{branch_stub.shape}")

- branch_asm.stub(branch_stub) do

- case branch_stub.shape

- in Default

- branch_asm.jnz(branch_stub.target0.address)

- branch_asm.jmp(branch_stub.target1.address)

- in Next0

- branch_asm.jz(branch_stub.target1.address)

- in Next1

- branch_asm.jnz(branch_stub.target0.address)

- end

- end

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def branchunless(jit, ctx, asm)

- # Check for interrupts, but only on backward branches that may create loops

- jump_offset = jit.operand(0, signed: true)

- if jump_offset < 0

- jit_check_ints(jit, ctx, asm)

- end

-

- # Get the branch target instruction offsets

- next_pc = jit.pc + C.VALUE.size * jit.insn.len

- jump_pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset)

-

- val_type = ctx.get_opnd_type(StackOpnd[0])

- val_opnd = ctx.stack_pop(1)

-

- if (result = val_type.known_truthy) != nil

- target_pc = result ? next_pc : jump_pc

- jit_direct_jump(jit.iseq, target_pc, ctx, asm)

- else

- # This `test` sets ZF only for Qnil and Qfalse, which let jz jump.

- asm.test(val_opnd, ~Qnil)

-

- # Set stubs

- branch_stub = BranchStub.new(

- iseq: jit.iseq,

- shape: Default,

- target0: BranchTarget.new(ctx:, pc: jump_pc), # branch target

- target1: BranchTarget.new(ctx:, pc: next_pc), # fallthrough

- )

- branch_stub.target0.address = Assembler.new.then do |ocb_asm|

- @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true)

- @ocb.write(ocb_asm)

- end

- branch_stub.target1.address = Assembler.new.then do |ocb_asm|

- @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false)

- @ocb.write(ocb_asm)

- end

-

- # Jump to target0 on jz

- branch_stub.compile = compile_branchunless(branch_stub)

- branch_stub.compile.call(asm)

- end

-

- EndBlock

- end

-

- def compile_branchunless(branch_stub) # Proc escapes arguments in memory

- proc do |branch_asm|

- branch_asm.comment("branchunless #{branch_stub.shape}")

- branch_asm.stub(branch_stub) do

- case branch_stub.shape

- in Default

- branch_asm.jz(branch_stub.target0.address)

- branch_asm.jmp(branch_stub.target1.address)

- in Next0

- branch_asm.jnz(branch_stub.target1.address)

- in Next1

- branch_asm.jz(branch_stub.target0.address)

- end

- end

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def branchnil(jit, ctx, asm)

- # Check for interrupts, but only on backward branches that may create loops

- jump_offset = jit.operand(0, signed: true)

- if jump_offset < 0

- jit_check_ints(jit, ctx, asm)

- end

-

- # Get the branch target instruction offsets

- next_pc = jit.pc + C.VALUE.size * jit.insn.len

- jump_pc = jit.pc + C.VALUE.size * (jit.insn.len + jump_offset)

-

- val_type = ctx.get_opnd_type(StackOpnd[0])

- val_opnd = ctx.stack_pop(1)

-

- if (result = val_type.known_nil) != nil

- target_pc = result ? jump_pc : next_pc

- jit_direct_jump(jit.iseq, target_pc, ctx, asm)

- else

- asm.cmp(val_opnd, Qnil)

-

- # Set stubs

- branch_stub = BranchStub.new(

- iseq: jit.iseq,

- shape: Default,

- target0: BranchTarget.new(ctx:, pc: jump_pc), # branch target

- target1: BranchTarget.new(ctx:, pc: next_pc), # fallthrough

- )

- branch_stub.target0.address = Assembler.new.then do |ocb_asm|

- @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true)

- @ocb.write(ocb_asm)

- end

- branch_stub.target1.address = Assembler.new.then do |ocb_asm|

- @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, false)

- @ocb.write(ocb_asm)

- end

-

- # Jump to target0 on je

- branch_stub.compile = compile_branchnil(branch_stub)

- branch_stub.compile.call(asm)

- end

-

- EndBlock

- end

-

- def compile_branchnil(branch_stub) # Proc escapes arguments in memory

- proc do |branch_asm|

- branch_asm.comment("branchnil #{branch_stub.shape}")

- branch_asm.stub(branch_stub) do

- case branch_stub.shape

- in Default

- branch_asm.je(branch_stub.target0.address)

- branch_asm.jmp(branch_stub.target1.address)

- in Next0

- branch_asm.jne(branch_stub.target1.address)

- in Next1

- branch_asm.je(branch_stub.target0.address)

- end

- end

- end

- end

-

- # once

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_case_dispatch(jit, ctx, asm)

- # Normally this instruction would lookup the key in a hash and jump to an

- # offset based on that.

- # Instead we can take the fallback case and continue with the next

- # instruction.

- # We'd hope that our jitted code will be sufficiently fast without the

- # hash lookup, at least for small hashes, but it's worth revisiting this

- # assumption in the future.

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

- starting_context = ctx.dup

-

- case_hash = jit.operand(0, ruby: true)

- else_offset = jit.operand(1)

-

- # Try to reorder case/else branches so that ones that are actually used come first.

- # Supporting only Fixnum for now so that the implementation can be an equality check.

- key_opnd = ctx.stack_pop(1)

- comptime_key = jit.peek_at_stack(0)

-

- # Check that all cases are fixnums to avoid having to register BOP assumptions on

- # all the types that case hashes support. This spends compile time to save memory.

- if fixnum?(comptime_key) && comptime_key <= 2**32 && C.rb_hash_keys(case_hash).all? { |key| fixnum?(key) }

- unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_EQQ)

- return CantCompile

- end

-

- # Check if the key is the same value

- asm.cmp(key_opnd, to_value(comptime_key))

- side_exit = side_exit(jit, starting_context)

- jit_chain_guard(:jne, jit, starting_context, asm, side_exit)

-

- # Get the offset for the compile-time key

- offset = C.rb_hash_stlike_lookup(case_hash, comptime_key)

- # NOTE: If we hit the else branch with various values, it could negatively impact the performance.

- jump_offset = offset || else_offset

-

- # Jump to the offset of case or else

- target_pc = jit.pc + (jit.insn.len + jump_offset) * C.VALUE.size

- jit_direct_jump(jit.iseq, target_pc, ctx, asm)

- EndBlock

- else

- KeepCompiling # continue with === branches

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_plus(jit, ctx, asm)

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- comptime_recv = jit.peek_at_stack(1)

- comptime_obj = jit.peek_at_stack(0)

-

- if fixnum?(comptime_recv) && fixnum?(comptime_obj)

- unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_PLUS)

- return CantCompile

- end

-

- # Check that both operands are fixnums

- guard_two_fixnums(jit, ctx, asm)

-

- obj_opnd = ctx.stack_pop

- recv_opnd = ctx.stack_pop

-

- asm.mov(:rax, recv_opnd)

- asm.sub(:rax, 1) # untag

- asm.mov(:rcx, obj_opnd)

- asm.add(:rax, :rcx)

- asm.jo(side_exit(jit, ctx))

-

- dst_opnd = ctx.stack_push(Type::Fixnum)

- asm.mov(dst_opnd, :rax)

-

- KeepCompiling

- else

- opt_send_without_block(jit, ctx, asm)

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_minus(jit, ctx, asm)

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- comptime_recv = jit.peek_at_stack(1)

- comptime_obj = jit.peek_at_stack(0)

-

- if fixnum?(comptime_recv) && fixnum?(comptime_obj)

- unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_MINUS)

- return CantCompile

- end

-

- # Check that both operands are fixnums

- guard_two_fixnums(jit, ctx, asm)

-

- obj_opnd = ctx.stack_pop

- recv_opnd = ctx.stack_pop

-

- asm.mov(:rax, recv_opnd)

- asm.mov(:rcx, obj_opnd)

- asm.sub(:rax, :rcx)

- asm.jo(side_exit(jit, ctx))

- asm.add(:rax, 1) # re-tag

-

- dst_opnd = ctx.stack_push(Type::Fixnum)

- asm.mov(dst_opnd, :rax)

-

- KeepCompiling

- else

- opt_send_without_block(jit, ctx, asm)

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_mult(jit, ctx, asm)

- opt_send_without_block(jit, ctx, asm)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_div(jit, ctx, asm)

- opt_send_without_block(jit, ctx, asm)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_mod(jit, ctx, asm)

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- if two_fixnums_on_stack?(jit)

- unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_MOD)

- return CantCompile

- end

-

- # Check that both operands are fixnums

- guard_two_fixnums(jit, ctx, asm)

-

- # Get the operands and destination from the stack

- arg1 = ctx.stack_pop(1)

- arg0 = ctx.stack_pop(1)

-

- # Check for arg0 % 0

- asm.cmp(arg1, 0)

- asm.je(side_exit(jit, ctx))

-

- # Call rb_fix_mod_fix(VALUE recv, VALUE obj)

- asm.mov(C_ARGS[0], arg0)

- asm.mov(C_ARGS[1], arg1)

- asm.call(C.rb_fix_mod_fix)

-

- # Push the return value onto the stack

- stack_ret = ctx.stack_push(Type::Fixnum)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- else

- opt_send_without_block(jit, ctx, asm)

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_eq(jit, ctx, asm)

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- if jit_equality_specialized(jit, ctx, asm, true)

- jump_to_next_insn(jit, ctx, asm)

- EndBlock

- else

- opt_send_without_block(jit, ctx, asm)

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_neq(jit, ctx, asm)

- # opt_neq is passed two rb_call_data as arguments:

- # first for ==, second for !=

- neq_cd = C.rb_call_data.new(jit.operand(1))

- opt_send_without_block(jit, ctx, asm, cd: neq_cd)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_lt(jit, ctx, asm)

- jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovl, bop: C::BOP_LT)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_le(jit, ctx, asm)

- jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovle, bop: C::BOP_LE)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_gt(jit, ctx, asm)

- jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovg, bop: C::BOP_GT)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_ge(jit, ctx, asm)

- jit_fixnum_cmp(jit, ctx, asm, opcode: :cmovge, bop: C::BOP_GE)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_ltlt(jit, ctx, asm)

- opt_send_without_block(jit, ctx, asm)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_and(jit, ctx, asm)

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- if two_fixnums_on_stack?(jit)

- unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_AND)

- return CantCompile

- end

-

- # Check that both operands are fixnums

- guard_two_fixnums(jit, ctx, asm)

-

- # Get the operands and destination from the stack

- arg1 = ctx.stack_pop(1)

- arg0 = ctx.stack_pop(1)

-

- asm.comment('bitwise and')

- asm.mov(:rax, arg0)

- asm.and(:rax, arg1)

-

- # Push the return value onto the stack

- dst = ctx.stack_push(Type::Fixnum)

- asm.mov(dst, :rax)

-

- KeepCompiling

- else

- opt_send_without_block(jit, ctx, asm)

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_or(jit, ctx, asm)

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- if two_fixnums_on_stack?(jit)

- unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_OR)

- return CantCompile

- end

-

- # Check that both operands are fixnums

- guard_two_fixnums(jit, ctx, asm)

-

- # Get the operands and destination from the stack

- asm.comment('bitwise or')

- arg1 = ctx.stack_pop(1)

- arg0 = ctx.stack_pop(1)

-

- # Do the bitwise or arg0 | arg1

- asm.mov(:rax, arg0)

- asm.or(:rax, arg1)

-

- # Push the return value onto the stack

- dst = ctx.stack_push(Type::Fixnum)

- asm.mov(dst, :rax)

-

- KeepCompiling

- else

- opt_send_without_block(jit, ctx, asm)

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_aref(jit, ctx, asm)

- cd = C.rb_call_data.new(jit.operand(0))

- argc = C.vm_ci_argc(cd.ci)

-

- if argc != 1

- asm.incr_counter(:optaref_argc_not_one)

- return CantCompile

- end

-

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- comptime_recv = jit.peek_at_stack(1)

- comptime_obj = jit.peek_at_stack(0)

-

- side_exit = side_exit(jit, ctx)

-

- if C.rb_class_of(comptime_recv) == Array && fixnum?(comptime_obj)

- unless Invariants.assume_bop_not_redefined(jit, C::ARRAY_REDEFINED_OP_FLAG, C::BOP_AREF)

- return CantCompile

- end

-

- idx_opnd = ctx.stack_opnd(0)

- recv_opnd = ctx.stack_opnd(1)

-

- not_array_exit = counted_exit(side_exit, :optaref_recv_not_array)

- jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv_opnd, StackOpnd[1], comptime_recv, not_array_exit)

-

- # Bail if idx is not a FIXNUM

- asm.mov(:rax, idx_opnd)

- asm.test(:rax, C::RUBY_FIXNUM_FLAG)

- asm.jz(counted_exit(side_exit, :optaref_arg_not_fixnum))

-

- # Call VALUE rb_ary_entry_internal(VALUE ary, long offset).

- # It never raises or allocates, so we don't need to write to cfp->pc.

- asm.sar(:rax, 1) # Convert fixnum to int

- asm.mov(C_ARGS[0], recv_opnd)

- asm.mov(C_ARGS[1], :rax)

- asm.call(C.rb_ary_entry_internal)

-

- # Pop the argument and the receiver

- ctx.stack_pop(2)

-

- # Push the return value onto the stack

- stack_ret = ctx.stack_push(Type::Unknown)

- asm.mov(stack_ret, C_RET)

-

- # Let guard chains share the same successor

- jump_to_next_insn(jit, ctx, asm)

- EndBlock

- elsif C.rb_class_of(comptime_recv) == Hash

- unless Invariants.assume_bop_not_redefined(jit, C::HASH_REDEFINED_OP_FLAG, C::BOP_AREF)

- return CantCompile

- end

-

- recv_opnd = ctx.stack_opnd(1)

-

- # Guard that the receiver is a Hash

- not_hash_exit = counted_exit(side_exit, :optaref_recv_not_hash)

- jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv_opnd, StackOpnd[1], comptime_recv, not_hash_exit)

-

- # Prepare to call rb_hash_aref(). It might call #hash on the key.

- jit_prepare_routine_call(jit, ctx, asm)

-

- asm.comment('call rb_hash_aref')

- key_opnd = ctx.stack_opnd(0)

- recv_opnd = ctx.stack_opnd(1)

- asm.mov(:rdi, recv_opnd)

- asm.mov(:rsi, key_opnd)

- asm.call(C.rb_hash_aref)

-

- # Pop the key and the receiver

- ctx.stack_pop(2)

-

- stack_ret = ctx.stack_push(Type::Unknown)

- asm.mov(stack_ret, C_RET)

-

- # Let guard chains share the same successor

- jump_to_next_insn(jit, ctx, asm)

- EndBlock

- else

- opt_send_without_block(jit, ctx, asm)

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_aset(jit, ctx, asm)

- # Defer compilation so we can specialize on a runtime `self`

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- comptime_recv = jit.peek_at_stack(2)

- comptime_key = jit.peek_at_stack(1)

-

- # Get the operands from the stack

- recv = ctx.stack_opnd(2)

- key = ctx.stack_opnd(1)

- _val = ctx.stack_opnd(0)

-

- if C.rb_class_of(comptime_recv) == Array && fixnum?(comptime_key)

- side_exit = side_exit(jit, ctx)

-

- # Guard receiver is an Array

- jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv, StackOpnd[2], comptime_recv, side_exit)

-

- # Guard key is a fixnum

- jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_key), key, StackOpnd[1], comptime_key, side_exit)

-

- # We might allocate or raise

- jit_prepare_routine_call(jit, ctx, asm)

-

- asm.comment('call rb_ary_store')

- recv = ctx.stack_opnd(2)

- key = ctx.stack_opnd(1)

- val = ctx.stack_opnd(0)

- asm.mov(:rax, key)

- asm.sar(:rax, 1) # FIX2LONG(key)

- asm.mov(C_ARGS[0], recv)

- asm.mov(C_ARGS[1], :rax)

- asm.mov(C_ARGS[2], val)

- asm.call(C.rb_ary_store)

-

- # rb_ary_store returns void

- # stored value should still be on stack

- val = ctx.stack_opnd(0)

-

- # Push the return value onto the stack

- ctx.stack_pop(3)

- stack_ret = ctx.stack_push(Type::Unknown)

- asm.mov(:rax, val)

- asm.mov(stack_ret, :rax)

-

- jump_to_next_insn(jit, ctx, asm)

- EndBlock

- elsif C.rb_class_of(comptime_recv) == Hash

- side_exit = side_exit(jit, ctx)

-

- # Guard receiver is a Hash

- jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_recv), recv, StackOpnd[2], comptime_recv, side_exit)

-

- # We might allocate or raise

- jit_prepare_routine_call(jit, ctx, asm)

-

- # Call rb_hash_aset

- recv = ctx.stack_opnd(2)

- key = ctx.stack_opnd(1)

- val = ctx.stack_opnd(0)

- asm.mov(C_ARGS[0], recv)

- asm.mov(C_ARGS[1], key)

- asm.mov(C_ARGS[2], val)

- asm.call(C.rb_hash_aset)

-

- # Push the return value onto the stack

- ctx.stack_pop(3)

- stack_ret = ctx.stack_push(Type::Unknown)

- asm.mov(stack_ret, C_RET)

-

- jump_to_next_insn(jit, ctx, asm)

- EndBlock

- else

- opt_send_without_block(jit, ctx, asm)

- end

- end

-

- # opt_aset_with

- # opt_aref_with

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_length(jit, ctx, asm)

- opt_send_without_block(jit, ctx, asm)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_size(jit, ctx, asm)

- opt_send_without_block(jit, ctx, asm)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_empty_p(jit, ctx, asm)

- opt_send_without_block(jit, ctx, asm)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_succ(jit, ctx, asm)

- opt_send_without_block(jit, ctx, asm)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_not(jit, ctx, asm)

- opt_send_without_block(jit, ctx, asm)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_regexpmatch2(jit, ctx, asm)

- opt_send_without_block(jit, ctx, asm)

- end

-

- # invokebuiltin

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_invokebuiltin_delegate(jit, ctx, asm)

- bf = C.rb_builtin_function.new(jit.operand(0))

- bf_argc = bf.argc

- start_index = jit.operand(1)

-

- # ec, self, and arguments

- if bf_argc + 2 > C_ARGS.size

- return CantCompile

- end

-

- # If the calls don't allocate, do they need up to date PC, SP?

- jit_prepare_routine_call(jit, ctx, asm)

-

- # Call the builtin func (ec, recv, arg1, arg2, ...)

- asm.comment('call builtin func')

- asm.mov(C_ARGS[0], EC)

- asm.mov(C_ARGS[1], [CFP, C.rb_control_frame_t.offsetof(:self)])

-

- # Copy arguments from locals

- if bf_argc > 0

- # Load environment pointer EP from CFP

- asm.mov(:rax, [CFP, C.rb_control_frame_t.offsetof(:ep)])

-

- bf_argc.times do |i|

- table_size = jit.iseq.body.local_table_size

- offs = -table_size - C::VM_ENV_DATA_SIZE + 1 + start_index + i

- asm.mov(C_ARGS[2 + i], [:rax, offs * C.VALUE.size])

- end

- end

- asm.call(bf.func_ptr)

-

- # Push the return value

- stack_ret = ctx.stack_push(Type::Unknown)

- asm.mov(stack_ret, C_RET)

-

- KeepCompiling

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def opt_invokebuiltin_delegate_leave(jit, ctx, asm)

- opt_invokebuiltin_delegate(jit, ctx, asm)

- # opt_invokebuiltin_delegate is always followed by leave insn

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def putobject_INT2FIX_0_(jit, ctx, asm)

- putobject(jit, ctx, asm, val: C.to_value(0))

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def putobject_INT2FIX_1_(jit, ctx, asm)

- putobject(jit, ctx, asm, val: C.to_value(1))

- end

-

- #

- # C func

- #

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_true(jit, ctx, asm, argc, _known_recv_class)

- return false if argc != 0

- asm.comment('nil? == true')

- ctx.stack_pop(1)

- stack_ret = ctx.stack_push(Type::True)

- asm.mov(stack_ret, Qtrue)

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_false(jit, ctx, asm, argc, _known_recv_class)

- return false if argc != 0

- asm.comment('nil? == false')

- ctx.stack_pop(1)

- stack_ret = ctx.stack_push(Type::False)

- asm.mov(stack_ret, Qfalse)

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_kernel_is_a(jit, ctx, asm, argc, known_recv_class)

- if argc != 1

- return false

- end

-

- # If this is a super call we might not know the class

- if known_recv_class.nil?

- return false

- end

-

- # Important note: The output code will simply `return true/false`.

- # Correctness follows from:

- # - `known_recv_class` implies there is a guard scheduled before here

- # for a particular `CLASS_OF(lhs)`.

- # - We guard that rhs is identical to the compile-time sample

- # - In general, for any two Class instances A, B, `A < B` does not change at runtime.

- # Class#superclass is stable.

-

- sample_rhs = jit.peek_at_stack(0)

- sample_lhs = jit.peek_at_stack(1)

-

- # We are not allowing module here because the module hierarchy can change at runtime.

- if C.RB_TYPE_P(sample_rhs, C::RUBY_T_CLASS)

- return false

- end

- sample_is_a = C.obj_is_kind_of(sample_lhs, sample_rhs)

-

- side_exit = side_exit(jit, ctx)

- asm.comment('Kernel#is_a?')

- asm.mov(:rax, to_value(sample_rhs))

- asm.cmp(ctx.stack_opnd(0), :rax)

- asm.jne(counted_exit(side_exit, :send_is_a_class_mismatch))

-

- ctx.stack_pop(2)

-

- if sample_is_a

- stack_ret = ctx.stack_push(Type::True)

- asm.mov(stack_ret, Qtrue)

- else

- stack_ret = ctx.stack_push(Type::False)

- asm.mov(stack_ret, Qfalse)

- end

- return true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_kernel_instance_of(jit, ctx, asm, argc, known_recv_class)

- if argc != 1

- return false

- end

-

- # If this is a super call we might not know the class

- if known_recv_class.nil?

- return false

- end

-

- # Important note: The output code will simply `return true/false`.

- # Correctness follows from:

- # - `known_recv_class` implies there is a guard scheduled before here

- # for a particular `CLASS_OF(lhs)`.

- # - We guard that rhs is identical to the compile-time sample

- # - For a particular `CLASS_OF(lhs)`, `rb_obj_class(lhs)` does not change.

- # (because for any singleton class `s`, `s.superclass.equal?(s.attached_object.class)`)

-

- sample_rhs = jit.peek_at_stack(0)

- sample_lhs = jit.peek_at_stack(1)

-

- # Filters out cases where the C implementation raises

- unless C.RB_TYPE_P(sample_rhs, C::RUBY_T_CLASS) || C.RB_TYPE_P(sample_rhs, C::RUBY_T_MODULE)

- return false

- end

-

- # We need to grab the class here to deal with singleton classes.

- # Instance of grabs the "real class" of the object rather than the

- # singleton class.

- sample_lhs_real_class = C.rb_obj_class(sample_lhs)

-

- sample_instance_of = (sample_lhs_real_class == sample_rhs)

-

- side_exit = side_exit(jit, ctx)

- asm.comment('Kernel#instance_of?')

- asm.mov(:rax, to_value(sample_rhs))

- asm.cmp(ctx.stack_opnd(0), :rax)

- asm.jne(counted_exit(side_exit, :send_instance_of_class_mismatch))

-

- ctx.stack_pop(2)

-

- if sample_instance_of

- stack_ret = ctx.stack_push(Type::True)

- asm.mov(stack_ret, Qtrue)

- else

- stack_ret = ctx.stack_push(Type::False)

- asm.mov(stack_ret, Qfalse)

- end

- return true;

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_obj_not(jit, ctx, asm, argc, _known_recv_class)

- return false if argc != 0

- recv_type = ctx.get_opnd_type(StackOpnd[0])

-

- case recv_type.known_truthy

- in false

- asm.comment('rb_obj_not(nil_or_false)')

- ctx.stack_pop(1)

- out_opnd = ctx.stack_push(Type::True)

- asm.mov(out_opnd, Qtrue)

- in true

- # Note: recv_type != Type::Nil && recv_type != Type::False.

- asm.comment('rb_obj_not(truthy)')

- ctx.stack_pop(1)

- out_opnd = ctx.stack_push(Type::False)

- asm.mov(out_opnd, Qfalse)

- in nil

- asm.comment('rb_obj_not')

-

- recv = ctx.stack_pop

- # This `test` sets ZF only for Qnil and Qfalse, which let cmovz set.

- asm.test(recv, ~Qnil)

- asm.mov(:rax, Qfalse)

- asm.mov(:rcx, Qtrue)

- asm.cmovz(:rax, :rcx)

-

- stack_ret = ctx.stack_push(Type::UnknownImm)

- asm.mov(stack_ret, :rax)

- end

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_obj_equal(jit, ctx, asm, argc, _known_recv_class)

- return false if argc != 1

- asm.comment('equal?')

- obj1 = ctx.stack_pop(1)

- obj2 = ctx.stack_pop(1)

-

- asm.mov(:rax, obj1)

- asm.mov(:rcx, obj2)

- asm.cmp(:rax, :rcx)

- asm.mov(:rax, Qfalse)

- asm.mov(:rcx, Qtrue)

- asm.cmove(:rax, :rcx)

-

- stack_ret = ctx.stack_push(Type::UnknownImm)

- asm.mov(stack_ret, :rax)

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_obj_not_equal(jit, ctx, asm, argc, _known_recv_class)

- return false if argc != 1

- jit_equality_specialized(jit, ctx, asm, false)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_mod_eqq(jit, ctx, asm, argc, _known_recv_class)

- return false if argc != 1

-

- asm.comment('Module#===')

- # By being here, we know that the receiver is a T_MODULE or a T_CLASS, because Module#=== can

- # only live on these objects. With that, we can call rb_obj_is_kind_of() without

- # jit_prepare_routine_call() or a control frame push because it can't raise, allocate, or call

- # Ruby methods with these inputs.

- # Note the difference in approach from Kernel#is_a? because we don't get a free guard for the

- # right hand side.

- lhs = ctx.stack_opnd(1) # the module

- rhs = ctx.stack_opnd(0)

- asm.mov(C_ARGS[0], rhs);

- asm.mov(C_ARGS[1], lhs);

- asm.call(C.rb_obj_is_kind_of)

-

- # Return the result

- ctx.stack_pop(2)

- stack_ret = ctx.stack_push(Type::UnknownImm)

- asm.mov(stack_ret, C_RET)

-

- return true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_int_equal(jit, ctx, asm, argc, _known_recv_class)

- return false if argc != 1

- return false unless two_fixnums_on_stack?(jit)

-

- guard_two_fixnums(jit, ctx, asm)

-

- # Compare the arguments

- asm.comment('rb_int_equal')

- arg1 = ctx.stack_pop(1)

- arg0 = ctx.stack_pop(1)

- asm.mov(:rax, arg1)

- asm.cmp(arg0, :rax)

- asm.mov(:rax, Qfalse)

- asm.mov(:rcx, Qtrue)

- asm.cmove(:rax, :rcx)

-

- stack_ret = ctx.stack_push(Type::UnknownImm)

- asm.mov(stack_ret, :rax)

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_int_mul(jit, ctx, asm, argc, _known_recv_class)

- return false if argc != 1

- return false unless two_fixnums_on_stack?(jit)

-

- guard_two_fixnums(jit, ctx, asm)

-

- asm.comment('rb_int_mul')

- y_opnd = ctx.stack_pop

- x_opnd = ctx.stack_pop

- asm.mov(C_ARGS[0], x_opnd)

- asm.mov(C_ARGS[1], y_opnd)

- asm.call(C.rb_fix_mul_fix)

-

- ret_opnd = ctx.stack_push(Type::Unknown)

- asm.mov(ret_opnd, C_RET)

- true

- end

-

- def jit_rb_int_div(jit, ctx, asm, argc, _known_recv_class)

- return false if argc != 1

- return false unless two_fixnums_on_stack?(jit)

-

- guard_two_fixnums(jit, ctx, asm)

-

- asm.comment('rb_int_div')

- y_opnd = ctx.stack_pop

- x_opnd = ctx.stack_pop

- asm.mov(:rax, y_opnd)

- asm.cmp(:rax, C.to_value(0))

- asm.je(side_exit(jit, ctx))

-

- asm.mov(C_ARGS[0], x_opnd)

- asm.mov(C_ARGS[1], :rax)

- asm.call(C.rb_fix_div_fix)

-

- ret_opnd = ctx.stack_push(Type::Unknown)

- asm.mov(ret_opnd, C_RET)

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_int_aref(jit, ctx, asm, argc, _known_recv_class)

- return false if argc != 1

- return false unless two_fixnums_on_stack?(jit)

-

- guard_two_fixnums(jit, ctx, asm)

-

- asm.comment('rb_int_aref')

- y_opnd = ctx.stack_pop

- x_opnd = ctx.stack_pop

-

- asm.mov(C_ARGS[0], x_opnd)

- asm.mov(C_ARGS[1], y_opnd)

- asm.call(C.rb_fix_aref)

-

- ret_opnd = ctx.stack_push(Type::UnknownImm)

- asm.mov(ret_opnd, C_RET)

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_str_empty_p(jit, ctx, asm, argc, known_recv_class)

- recv_opnd = ctx.stack_pop(1)

- out_opnd = ctx.stack_push(Type::UnknownImm)

-

- asm.comment('get string length')

- asm.mov(:rax, recv_opnd)

- str_len_opnd = [:rax, C.RString.offsetof(:len)]

-

- asm.cmp(str_len_opnd, 0)

- asm.mov(:rax, Qfalse)

- asm.mov(:rcx, Qtrue)

- asm.cmove(:rax, :rcx)

- asm.mov(out_opnd, :rax)

-

- return true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_str_to_s(jit, ctx, asm, argc, known_recv_class)

- return false if argc != 0

- if known_recv_class == String

- asm.comment('to_s on plain string')

- # The method returns the receiver, which is already on the stack.

- # No stack movement.

- return true

- end

- false

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_str_bytesize(jit, ctx, asm, argc, known_recv_class)

- asm.comment('String#bytesize')

-

- recv = ctx.stack_pop(1)

- asm.mov(C_ARGS[0], recv)

- asm.call(C.rb_str_bytesize)

-

- out_opnd = ctx.stack_push(Type::Fixnum)

- asm.mov(out_opnd, C_RET)

-

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_str_concat(jit, ctx, asm, argc, known_recv_class)

- # The << operator can accept integer codepoints for characters

- # as the argument. We only specially optimise string arguments.

- # If the peeked-at compile time argument is something other than

- # a string, assume it won't be a string later either.

- comptime_arg = jit.peek_at_stack(0)

- unless C.RB_TYPE_P(comptime_arg, C::RUBY_T_STRING)

- return false

- end

-

- # Guard that the concat argument is a string

- asm.mov(:rax, ctx.stack_opnd(0))

- guard_object_is_string(jit, ctx, asm, :rax, :rcx, StackOpnd[0])

-

- # Guard buffers from GC since rb_str_buf_append may allocate. During the VM lock on GC,

- # other Ractors may trigger global invalidation, so we need ctx.clear_local_types.

- # PC is used on errors like Encoding::CompatibilityError raised by rb_str_buf_append.

- jit_prepare_routine_call(jit, ctx, asm)

-

- concat_arg = ctx.stack_pop(1)

- recv = ctx.stack_pop(1)

-

- # Test if string encodings differ. If different, use rb_str_append. If the same,

- # use rb_yjit_str_simple_append, which calls rb_str_cat.

- asm.comment('<< on strings')

-

- # Take receiver's object flags XOR arg's flags. If any

- # string-encoding flags are different between the two,

- # the encodings don't match.

- recv_reg = :rax

- asm.mov(recv_reg, recv)

- concat_arg_reg = :rcx

- asm.mov(concat_arg_reg, concat_arg)

- asm.mov(recv_reg, [recv_reg, C.RBasic.offsetof(:flags)])

- asm.mov(concat_arg_reg, [concat_arg_reg, C.RBasic.offsetof(:flags)])

- asm.xor(recv_reg, concat_arg_reg)

- asm.test(recv_reg, C::RUBY_ENCODING_MASK)

-

- # Push once, use the resulting operand in both branches below.

- stack_ret = ctx.stack_push(Type::TString)

-

- enc_mismatch = asm.new_label('enc_mismatch')

- asm.jnz(enc_mismatch)

-

- # If encodings match, call the simple append function and jump to return

- asm.mov(C_ARGS[0], recv)

- asm.mov(C_ARGS[1], concat_arg)

- asm.call(C.rjit_str_simple_append)

- ret_label = asm.new_label('func_return')

- asm.mov(stack_ret, C_RET)

- asm.jmp(ret_label)

-

- # If encodings are different, use a slower encoding-aware concatenate

- asm.write_label(enc_mismatch)

- asm.mov(C_ARGS[0], recv)

- asm.mov(C_ARGS[1], concat_arg)

- asm.call(C.rb_str_buf_append)

- asm.mov(stack_ret, C_RET)

- # Drop through to return

-

- asm.write_label(ret_label)

-

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_str_uplus(jit, ctx, asm, argc, _known_recv_class)

- if argc != 0

- return false

- end

-

- # We allocate when we dup the string

- jit_prepare_routine_call(jit, ctx, asm)

-

- asm.comment('Unary plus on string')

- asm.mov(:rax, ctx.stack_pop(1)) # recv_opnd

- asm.mov(:rcx, [:rax, C.RBasic.offsetof(:flags)]) # flags_opnd

- asm.test(:rcx, C::RUBY_FL_FREEZE)

-

- ret_label = asm.new_label('stack_ret')

-

- # String#+@ can only exist on T_STRING

- stack_ret = ctx.stack_push(Type::TString)

-

- # If the string isn't frozen, we just return it.

- asm.mov(stack_ret, :rax) # recv_opnd

- asm.jz(ret_label)

-

- # Str is frozen - duplicate it

- asm.mov(C_ARGS[0], :rax) # recv_opnd

- asm.call(C.rb_str_dup)

- asm.mov(stack_ret, C_RET)

-

- asm.write_label(ret_label)

-

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_str_getbyte(jit, ctx, asm, argc, _known_recv_class)

- return false if argc != 1

- asm.comment('rb_str_getbyte')

-

- index_opnd = ctx.stack_pop

- str_opnd = ctx.stack_pop

- asm.mov(C_ARGS[0], str_opnd)

- asm.mov(C_ARGS[1], index_opnd)

- asm.call(C.rb_str_getbyte)

-

- ret_opnd = ctx.stack_push(Type::Fixnum)

- asm.mov(ret_opnd, C_RET)

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_ary_empty_p(jit, ctx, asm, argc, _known_recv_class)

- array_reg = :rax

- asm.mov(array_reg, ctx.stack_pop(1))

- jit_array_len(asm, array_reg, :rcx)

-

- asm.test(:rcx, :rcx)

- asm.mov(:rax, Qfalse)

- asm.mov(:rcx, Qtrue)

- asm.cmovz(:rax, :rcx)

-

- out_opnd = ctx.stack_push(Type::UnknownImm)

- asm.mov(out_opnd, :rax)

-

- return true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_ary_push(jit, ctx, asm, argc, _known_recv_class)

- return false if argc != 1

- asm.comment('rb_ary_push')

-

- jit_prepare_routine_call(jit, ctx, asm)

-

- item_opnd = ctx.stack_pop

- ary_opnd = ctx.stack_pop

- asm.mov(C_ARGS[0], ary_opnd)

- asm.mov(C_ARGS[1], item_opnd)

- asm.call(C.rb_ary_push)

-

- ret_opnd = ctx.stack_push(Type::TArray)

- asm.mov(ret_opnd, C_RET)

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_obj_respond_to(jit, ctx, asm, argc, known_recv_class)

- # respond_to(:sym) or respond_to(:sym, true)

- if argc != 1 && argc != 2

- return false

- end

-

- if known_recv_class.nil?

- return false

- end

-

- recv_class = known_recv_class

-

- # Get the method_id from compile time. We will later add a guard against it.

- mid_sym = jit.peek_at_stack(argc - 1)

- unless static_symbol?(mid_sym)

- return false

- end

- mid = C.rb_sym2id(mid_sym)

-

- # This represents the value of the "include_all" argument and whether it's known

- allow_priv = if argc == 1

- # Default is false

- false

- else

- # Get value from type information (may or may not be known)

- ctx.get_opnd_type(StackOpnd[0]).known_truthy

- end

-

- target_cme = C.rb_callable_method_entry_or_negative(recv_class, mid)

-

- # Should never be null, as in that case we will be returned a "negative CME"

- assert_equal(false, target_cme.nil?)

-

- cme_def_type = C.UNDEFINED_METHOD_ENTRY_P(target_cme) ? C::VM_METHOD_TYPE_UNDEF : target_cme.def.type

-

- if cme_def_type == C::VM_METHOD_TYPE_REFINED

- return false

- end

-

- visibility = if cme_def_type == C::VM_METHOD_TYPE_UNDEF

- C::METHOD_VISI_UNDEF

- else

- C.METHOD_ENTRY_VISI(target_cme)

- end

-

- result =

- case [visibility, allow_priv]

- in C::METHOD_VISI_UNDEF, _ then Qfalse # No method => false

- in C::METHOD_VISI_PUBLIC, _ then Qtrue # Public method => true regardless of include_all

- in _, true then Qtrue # include_all => always true

- else return false # not public and include_all not known, can't compile

- end

-

- if result != Qtrue

- # Only if respond_to_missing? hasn't been overridden

- # In the future, we might want to jit the call to respond_to_missing?

- unless Invariants.assume_method_basic_definition(jit, recv_class, C.idRespond_to_missing)

- return false

- end

- end

-

- # Invalidate this block if method lookup changes for the method being queried. This works

- # both for the case where a method does or does not exist, as for the latter we asked for a

- # "negative CME" earlier.

- Invariants.assume_method_lookup_stable(jit, target_cme)

-

- # Generate a side exit

- side_exit = side_exit(jit, ctx)

-

- if argc == 2

- # pop include_all argument (we only use its type info)

- ctx.stack_pop(1)

- end

-

- sym_opnd = ctx.stack_pop(1)

- _recv_opnd = ctx.stack_pop(1)

-

- # This is necessary because we have no guarantee that sym_opnd is a constant

- asm.comment('guard known mid')

- asm.mov(:rax, to_value(mid_sym))

- asm.cmp(sym_opnd, :rax)

- asm.jne(side_exit)

-

- putobject(jit, ctx, asm, val: result)

-

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_rb_f_block_given_p(jit, ctx, asm, argc, _known_recv_class)

- asm.comment('block_given?')

-

- # Same as rb_vm_frame_block_handler

- jit_get_lep(jit, asm, reg: :rax)

- asm.mov(:rax, [:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler

-

- ctx.stack_pop(1)

- out_opnd = ctx.stack_push(Type::UnknownImm)

-

- # Return `block_handler != VM_BLOCK_HANDLER_NONE`

- asm.cmp(:rax, C::VM_BLOCK_HANDLER_NONE)

- asm.mov(:rax, Qfalse)

- asm.mov(:rcx, Qtrue)

- asm.cmovne(:rax, :rcx) # block_given

- asm.mov(out_opnd, :rax)

-

- true

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_thread_s_current(jit, ctx, asm, argc, _known_recv_class)

- return false if argc != 0

- asm.comment('Thread.current')

- ctx.stack_pop(1)

-

- # ec->thread_ptr

- asm.mov(:rax, [EC, C.rb_execution_context_t.offsetof(:thread_ptr)])

-

- # thread->self

- asm.mov(:rax, [:rax, C.rb_thread_struct.offsetof(:self)])

-

- stack_ret = ctx.stack_push(Type::UnknownHeap)

- asm.mov(stack_ret, :rax)

- true

- end

-

- #

- # Helpers

- #

-

- def register_cfunc_codegen_funcs

- # Specialization for C methods. See register_cfunc_method for details.

- register_cfunc_method(BasicObject, :!, :jit_rb_obj_not)

-

- register_cfunc_method(NilClass, :nil?, :jit_rb_true)

- register_cfunc_method(Kernel, :nil?, :jit_rb_false)

- register_cfunc_method(Kernel, :is_a?, :jit_rb_kernel_is_a)

- register_cfunc_method(Kernel, :kind_of?, :jit_rb_kernel_is_a)

- register_cfunc_method(Kernel, :instance_of?, :jit_rb_kernel_instance_of)

-

- register_cfunc_method(BasicObject, :==, :jit_rb_obj_equal)

- register_cfunc_method(BasicObject, :equal?, :jit_rb_obj_equal)

- register_cfunc_method(BasicObject, :!=, :jit_rb_obj_not_equal)

- register_cfunc_method(Kernel, :eql?, :jit_rb_obj_equal)

- register_cfunc_method(Module, :==, :jit_rb_obj_equal)

- register_cfunc_method(Module, :===, :jit_rb_mod_eqq)

- register_cfunc_method(Symbol, :==, :jit_rb_obj_equal)

- register_cfunc_method(Symbol, :===, :jit_rb_obj_equal)

- register_cfunc_method(Integer, :==, :jit_rb_int_equal)

- register_cfunc_method(Integer, :===, :jit_rb_int_equal)

-

- # rb_str_to_s() methods in string.c

- register_cfunc_method(String, :empty?, :jit_rb_str_empty_p)

- register_cfunc_method(String, :to_s, :jit_rb_str_to_s)

- register_cfunc_method(String, :to_str, :jit_rb_str_to_s)

- register_cfunc_method(String, :bytesize, :jit_rb_str_bytesize)

- register_cfunc_method(String, :<<, :jit_rb_str_concat)

- register_cfunc_method(String, :+@, :jit_rb_str_uplus)

-

- # rb_ary_empty_p() method in array.c

- register_cfunc_method(Array, :empty?, :jit_rb_ary_empty_p)

-

- register_cfunc_method(Kernel, :respond_to?, :jit_obj_respond_to)

- register_cfunc_method(Kernel, :block_given?, :jit_rb_f_block_given_p)

-

- # Thread.current

- register_cfunc_method(C.rb_singleton_class(Thread), :current, :jit_thread_s_current)

-

- #---

- register_cfunc_method(Array, :<<, :jit_rb_ary_push)

- register_cfunc_method(Integer, :*, :jit_rb_int_mul)

- register_cfunc_method(Integer, :/, :jit_rb_int_div)

- register_cfunc_method(Integer, :[], :jit_rb_int_aref)

- register_cfunc_method(String, :getbyte, :jit_rb_str_getbyte)

- end

-

- def register_cfunc_method(klass, mid_sym, func)

- mid = C.rb_intern(mid_sym.to_s)

- me = C.rb_method_entry_at(klass, mid)

-

- assert_equal(false, me.nil?)

-

- # Only cfuncs are supported

- method_serial = me.def.method_serial

-

- @cfunc_codegen_table[method_serial] = method(func)

- end

-

- def lookup_cfunc_codegen(cme_def)

- @cfunc_codegen_table[cme_def.method_serial]

- end

-

- def stack_swap(_jit, ctx, asm, offset0, offset1)

- stack0_mem = ctx.stack_opnd(offset0)

- stack1_mem = ctx.stack_opnd(offset1)

-

- mapping0 = ctx.get_opnd_mapping(StackOpnd[offset0])

- mapping1 = ctx.get_opnd_mapping(StackOpnd[offset1])

-

- asm.mov(:rax, stack0_mem)

- asm.mov(:rcx, stack1_mem)

- asm.mov(stack0_mem, :rcx)

- asm.mov(stack1_mem, :rax)

-

- ctx.set_opnd_mapping(StackOpnd[offset0], mapping1)

- ctx.set_opnd_mapping(StackOpnd[offset1], mapping0)

- end

-

- def jit_getlocal_generic(jit, ctx, asm, idx:, level:)

- # Load environment pointer EP (level 0) from CFP

- ep_reg = :rax

- jit_get_ep(asm, level, reg: ep_reg)

-

- # Load the local from the block

- # val = *(vm_get_ep(GET_EP(), level) - idx);

- asm.mov(:rax, [ep_reg, -idx * C.VALUE.size])

-

- # Write the local at SP

- stack_top = if level == 0

- local_idx = ep_offset_to_local_idx(jit.iseq, idx)

- ctx.stack_push_local(local_idx)

- else

- ctx.stack_push(Type::Unknown)

- end

-

- asm.mov(stack_top, :rax)

- KeepCompiling

- end

-

- def jit_setlocal_generic(jit, ctx, asm, idx:, level:)

- value_type = ctx.get_opnd_type(StackOpnd[0])

-

- # Load environment pointer EP at level

- ep_reg = :rax

- jit_get_ep(asm, level, reg: ep_reg)

-

- # Write barriers may be required when VM_ENV_FLAG_WB_REQUIRED is set, however write barriers

- # only affect heap objects being written. If we know an immediate value is being written we

- # can skip this check.

- unless value_type.imm?

- # flags & VM_ENV_FLAG_WB_REQUIRED

- flags_opnd = [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_FLAGS]

- asm.test(flags_opnd, C::VM_ENV_FLAG_WB_REQUIRED)

-

- # if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0

- asm.jnz(side_exit(jit, ctx))

- end

-

- if level == 0

- local_idx = ep_offset_to_local_idx(jit.iseq, idx)

- ctx.set_local_type(local_idx, value_type)

- end

-

- # Pop the value to write from the stack

- stack_top = ctx.stack_pop(1)

-

- # Write the value at the environment pointer

- asm.mov(:rcx, stack_top)

- asm.mov([ep_reg, -(C.VALUE.size * idx)], :rcx)

-

- KeepCompiling

- end

-

- # Compute the index of a local variable from its slot index

- def ep_offset_to_local_idx(iseq, ep_offset)

- # Layout illustration

- # This is an array of VALUE

- # | VM_ENV_DATA_SIZE |

- # v v

- # low addr <+-------+-------+-------+-------+------------------+

- # |local 0|local 1| ... |local n| .... |

- # +-------+-------+-------+-------+------------------+

- # ^ ^ ^ ^

- # +-------+---local_table_size----+ cfp->ep--+

- # | |

- # +------------------ep_offset---------------+

- #

- # See usages of local_var_name() from iseq.c for similar calculation.

-

- # Equivalent of iseq->body->local_table_size

- local_table_size = iseq.body.local_table_size

- op = ep_offset - C::VM_ENV_DATA_SIZE

- local_idx = local_table_size - op - 1

- assert_equal(true, local_idx >= 0 && local_idx < local_table_size)

- local_idx

- end

-

- # Compute the index of a local variable from its slot index

- def slot_to_local_idx(iseq, slot_idx)

- # Layout illustration

- # This is an array of VALUE

- # | VM_ENV_DATA_SIZE |

- # v v

- # low addr <+-------+-------+-------+-------+------------------+

- # |local 0|local 1| ... |local n| .... |

- # +-------+-------+-------+-------+------------------+

- # ^ ^ ^ ^

- # +-------+---local_table_size----+ cfp->ep--+

- # | |

- # +------------------slot_idx----------------+

- #

- # See usages of local_var_name() from iseq.c for similar calculation.

-

- local_table_size = iseq.body.local_table_size

- op = slot_idx - C::VM_ENV_DATA_SIZE

- local_table_size - op - 1

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def guard_object_is_heap(jit, ctx, asm, object, object_opnd, counter = nil)

- object_type = ctx.get_opnd_type(object_opnd)

- if object_type.heap?

- return

- end

-

- side_exit = side_exit(jit, ctx)

- side_exit = counted_exit(side_exit, counter) if counter

-

- asm.comment('guard object is heap')

- # Test that the object is not an immediate

- asm.test(object, C::RUBY_IMMEDIATE_MASK)

- asm.jnz(side_exit)

-

- # Test that the object is not false

- asm.cmp(object, Qfalse)

- asm.je(side_exit)

-

- if object_type.diff(Type::UnknownHeap) != TypeDiff::Incompatible

- ctx.upgrade_opnd_type(object_opnd, Type::UnknownHeap)

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def guard_object_is_array(jit, ctx, asm, object_reg, flags_reg, object_opnd, counter = nil)

- object_type = ctx.get_opnd_type(object_opnd)

- if object_type.array?

- return

- end

-

- guard_object_is_heap(jit, ctx, asm, object_reg, object_opnd, counter)

-

- side_exit = side_exit(jit, ctx)

- side_exit = counted_exit(side_exit, counter) if counter

-

- asm.comment('guard object is array')

- # Pull out the type mask

- asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)])

- asm.and(flags_reg, C::RUBY_T_MASK)

-

- # Compare the result with T_ARRAY

- asm.cmp(flags_reg, C::RUBY_T_ARRAY)

- asm.jne(side_exit)

-

- if object_type.diff(Type::TArray) != TypeDiff::Incompatible

- ctx.upgrade_opnd_type(object_opnd, Type::TArray)

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def guard_object_is_string(jit, ctx, asm, object_reg, flags_reg, object_opnd, counter = nil)

- object_type = ctx.get_opnd_type(object_opnd)

- if object_type.string?

- return

- end

-

- guard_object_is_heap(jit, ctx, asm, object_reg, object_opnd, counter)

-

- side_exit = side_exit(jit, ctx)

- side_exit = counted_exit(side_exit, counter) if counter

-

- asm.comment('guard object is string')

- # Pull out the type mask

- asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)])

- asm.and(flags_reg, C::RUBY_T_MASK)

-

- # Compare the result with T_STRING

- asm.cmp(flags_reg, C::RUBY_T_STRING)

- asm.jne(side_exit)

-

- if object_type.diff(Type::TString) != TypeDiff::Incompatible

- ctx.upgrade_opnd_type(object_opnd, Type::TString)

- end

- end

-

- # clobbers object_reg

- def guard_object_is_not_ruby2_keyword_hash(asm, object_reg, flags_reg, side_exit)

- asm.comment('guard object is not ruby2 keyword hash')

-

- not_ruby2_keyword = asm.new_label('not_ruby2_keyword')

- asm.test(object_reg, C::RUBY_IMMEDIATE_MASK)

- asm.jnz(not_ruby2_keyword)

-

- asm.cmp(object_reg, Qfalse)

- asm.je(not_ruby2_keyword)

-

- asm.mov(flags_reg, [object_reg, C.RBasic.offsetof(:flags)])

- type_reg = object_reg

- asm.mov(type_reg, flags_reg)

- asm.and(type_reg, C::RUBY_T_MASK)

-

- asm.cmp(type_reg, C::RUBY_T_HASH)

- asm.jne(not_ruby2_keyword)

-

- asm.test(flags_reg, C::RHASH_PASS_AS_KEYWORDS)

- asm.jnz(side_exit)

-

- asm.write_label(not_ruby2_keyword)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_chain_guard(opcode, jit, ctx, asm, side_exit, limit: 20)

- opcode => :je | :jne | :jnz | :jz

-

- if ctx.chain_depth < limit

- deeper = ctx.dup

- deeper.chain_depth += 1

-

- branch_stub = BranchStub.new(

- iseq: jit.iseq,

- shape: Default,

- target0: BranchTarget.new(ctx: deeper, pc: jit.pc),

- )

- branch_stub.target0.address = Assembler.new.then do |ocb_asm|

- @exit_compiler.compile_branch_stub(deeper, ocb_asm, branch_stub, true)

- @ocb.write(ocb_asm)

- end

- branch_stub.compile = compile_jit_chain_guard(branch_stub, opcode:)

- branch_stub.compile.call(asm)

- else

- asm.public_send(opcode, side_exit)

- end

- end

-

- def compile_jit_chain_guard(branch_stub, opcode:) # Proc escapes arguments in memory

- proc do |branch_asm|

- # Not using `asm.comment` here since it's usually put before cmp/test before this.

- branch_asm.stub(branch_stub) do

- case branch_stub.shape

- in Default

- branch_asm.public_send(opcode, branch_stub.target0.address)

- end

- end

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_guard_known_klass(jit, ctx, asm, known_klass, obj_opnd, insn_opnd, comptime_obj, side_exit, limit: 10)

- # Only memory operand is supported for now

- assert_equal(true, obj_opnd.is_a?(Array))

-

- known_klass = C.to_value(known_klass)

- val_type = ctx.get_opnd_type(insn_opnd)

- if val_type.known_class == known_klass

- # We already know from type information that this is a match

- return

- end

-

- # Touching this as Ruby could crash for FrozenCore

- if known_klass == C.rb_cNilClass

- assert(!val_type.heap?)

- assert(val_type.unknown?)

-

- asm.comment('guard object is nil')

- asm.cmp(obj_opnd, Qnil)

- jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)

-

- ctx.upgrade_opnd_type(insn_opnd, Type::Nil)

- elsif known_klass == C.rb_cTrueClass

- assert(!val_type.heap?)

- assert(val_type.unknown?)

-

- asm.comment('guard object is true')

- asm.cmp(obj_opnd, Qtrue)

- jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)

-

- ctx.upgrade_opnd_type(insn_opnd, Type::True)

- elsif known_klass == C.rb_cFalseClass

- assert(!val_type.heap?)

- assert(val_type.unknown?)

-

- asm.comment('guard object is false')

- asm.cmp(obj_opnd, Qfalse)

- jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)

-

- ctx.upgrade_opnd_type(insn_opnd, Type::False)

- elsif known_klass == C.rb_cInteger && fixnum?(comptime_obj)

- # We will guard fixnum and bignum as though they were separate classes

- # BIGNUM can be handled by the general else case below

- assert(val_type.unknown?)

-

- asm.comment('guard object is fixnum')

- asm.test(obj_opnd, C::RUBY_FIXNUM_FLAG)

- jit_chain_guard(:jz, jit, ctx, asm, side_exit, limit:)

-

- ctx.upgrade_opnd_type(insn_opnd, Type::Fixnum)

- elsif known_klass == C.rb_cSymbol && static_symbol?(comptime_obj)

- assert(!val_type.heap?)

- # We will guard STATIC vs DYNAMIC as though they were separate classes

- # DYNAMIC symbols can be handled by the general else case below

- if val_type != Type::ImmSymbol || !val_type.imm?

- assert(val_type.unknown?)

-

- asm.comment('guard object is static symbol')

- assert_equal(8, C::RUBY_SPECIAL_SHIFT)

- asm.cmp(BytePtr[*obj_opnd], C::RUBY_SYMBOL_FLAG)

- jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)

-

- ctx.upgrade_opnd_type(insn_opnd, Type::ImmSymbol)

- end

- elsif known_klass == C.rb_cFloat && flonum?(comptime_obj)

- assert(!val_type.heap?)

- if val_type != Type::Flonum || !val_type.imm?

- assert(val_type.unknown?)

-

- # We will guard flonum vs heap float as though they were separate classes

- asm.comment('guard object is flonum')

- asm.mov(:rax, obj_opnd)

- asm.and(:rax, C::RUBY_FLONUM_MASK)

- asm.cmp(:rax, C::RUBY_FLONUM_FLAG)

- jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)

-

- ctx.upgrade_opnd_type(insn_opnd, Type::Flonum)

- end

- elsif C.RCLASS_SINGLETON_P(known_klass) && comptime_obj == C.rb_class_attached_object(known_klass)

- # Singleton classes are attached to one specific object, so we can

- # avoid one memory access (and potentially the is_heap check) by

- # looking for the expected object directly.

- # Note that in case the sample instance has a singleton class that

- # doesn't attach to the sample instance, it means the sample instance

- # has an empty singleton class that hasn't been materialized yet. In

- # this case, comparing against the sample instance doesn't guarantee

- # that its singleton class is empty, so we can't avoid the memory

- # access. As an example, `Object.new.singleton_class` is an object in

- # this situation.

- asm.comment('guard known object with singleton class')

- asm.mov(:rax, to_value(comptime_obj))

- asm.cmp(obj_opnd, :rax)

- jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)

- elsif val_type == Type::CString && known_klass == C.rb_cString

- # guard elided because the context says we've already checked

- assert_equal(C.to_value(C.rb_class_of(comptime_obj)), C.rb_cString)

- else

- assert(!val_type.imm?)

-

- # Load memory to a register

- asm.mov(:rax, obj_opnd)

- obj_opnd = :rax

-

- # Check that the receiver is a heap object

- # Note: if we get here, the class doesn't have immediate instances.

- unless val_type.heap?

- asm.comment('guard not immediate')

- asm.test(obj_opnd, C::RUBY_IMMEDIATE_MASK)

- jit_chain_guard(:jnz, jit, ctx, asm, side_exit, limit:)

- asm.cmp(obj_opnd, Qfalse)

- jit_chain_guard(:je, jit, ctx, asm, side_exit, limit:)

- end

-

- # Bail if receiver class is different from known_klass

- klass_opnd = [obj_opnd, C.RBasic.offsetof(:klass)]

- asm.comment("guard known class #{known_klass}")

- asm.mov(:rcx, known_klass)

- asm.cmp(klass_opnd, :rcx)

- jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:)

-

- if known_klass == C.rb_cString

- # Upgrading to Type::CString here is incorrect.

- # The guard we put only checks RBASIC_CLASS(obj),

- # which adding a singleton class can change. We

- # additionally need to know the string is frozen

- # to claim Type::CString.

- ctx.upgrade_opnd_type(insn_opnd, Type::TString)

- elsif known_klass == C.rb_cArray

- ctx.upgrade_opnd_type(insn_opnd, Type::TArray)

- end

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- def two_fixnums_on_stack?(jit)

- comptime_recv = jit.peek_at_stack(1)

- comptime_arg = jit.peek_at_stack(0)

- return fixnum?(comptime_recv) && fixnum?(comptime_arg)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def guard_two_fixnums(jit, ctx, asm)

- # Get stack operands without popping them

- arg1 = ctx.stack_opnd(0)

- arg0 = ctx.stack_opnd(1)

-

- # Get the stack operand types

- arg1_type = ctx.get_opnd_type(StackOpnd[0])

- arg0_type = ctx.get_opnd_type(StackOpnd[1])

-

- if arg0_type.heap? || arg1_type.heap?

- asm.comment('arg is heap object')

- asm.jmp(side_exit(jit, ctx))

- return

- end

-

- if arg0_type != Type::Fixnum && arg0_type.specific?

- asm.comment('arg0 not fixnum')

- asm.jmp(side_exit(jit, ctx))

- return

- end

-

- if arg1_type != Type::Fixnum && arg1_type.specific?

- asm.comment('arg1 not fixnum')

- asm.jmp(side_exit(jit, ctx))

- return

- end

-

- assert(!arg0_type.heap?)

- assert(!arg1_type.heap?)

- assert(arg0_type == Type::Fixnum || arg0_type.unknown?)

- assert(arg1_type == Type::Fixnum || arg1_type.unknown?)

-

- # If not fixnums at run-time, fall back

- if arg0_type != Type::Fixnum

- asm.comment('guard arg0 fixnum')

- asm.test(arg0, C::RUBY_FIXNUM_FLAG)

- jit_chain_guard(:jz, jit, ctx, asm, side_exit(jit, ctx))

- end

- if arg1_type != Type::Fixnum

- asm.comment('guard arg1 fixnum')

- asm.test(arg1, C::RUBY_FIXNUM_FLAG)

- jit_chain_guard(:jz, jit, ctx, asm, side_exit(jit, ctx))

- end

-

- # Set stack types in context

- ctx.upgrade_opnd_type(StackOpnd[0], Type::Fixnum)

- ctx.upgrade_opnd_type(StackOpnd[1], Type::Fixnum)

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_fixnum_cmp(jit, ctx, asm, opcode:, bop:)

- opcode => :cmovl | :cmovle | :cmovg | :cmovge

-

- unless jit.at_current_insn?

- defer_compilation(jit, ctx, asm)

- return EndBlock

- end

-

- comptime_recv = jit.peek_at_stack(1)

- comptime_obj = jit.peek_at_stack(0)

-

- if fixnum?(comptime_recv) && fixnum?(comptime_obj)

- unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, bop)

- return CantCompile

- end

-

- # Check that both operands are fixnums

- guard_two_fixnums(jit, ctx, asm)

-

- obj_opnd = ctx.stack_pop

- recv_opnd = ctx.stack_pop

-

- asm.mov(:rax, obj_opnd)

- asm.cmp(recv_opnd, :rax)

- asm.mov(:rax, Qfalse)

- asm.mov(:rcx, Qtrue)

- asm.public_send(opcode, :rax, :rcx)

-

- dst_opnd = ctx.stack_push(Type::UnknownImm)

- asm.mov(dst_opnd, :rax)

-

- KeepCompiling

- else

- opt_send_without_block(jit, ctx, asm)

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_equality_specialized(jit, ctx, asm, gen_eq)

- # Create a side-exit to fall back to the interpreter

- side_exit = side_exit(jit, ctx)

-

- a_opnd = ctx.stack_opnd(1)

- b_opnd = ctx.stack_opnd(0)

-

- comptime_a = jit.peek_at_stack(1)

- comptime_b = jit.peek_at_stack(0)

-

- if two_fixnums_on_stack?(jit)

- unless Invariants.assume_bop_not_redefined(jit, C::INTEGER_REDEFINED_OP_FLAG, C::BOP_EQ)

- return false

- end

-

- guard_two_fixnums(jit, ctx, asm)

-

- asm.comment('check fixnum equality')

- asm.mov(:rax, a_opnd)

- asm.mov(:rcx, b_opnd)

- asm.cmp(:rax, :rcx)

- asm.mov(:rax, gen_eq ? Qfalse : Qtrue)

- asm.mov(:rcx, gen_eq ? Qtrue : Qfalse)

- asm.cmove(:rax, :rcx)

-

- # Push the output on the stack

- ctx.stack_pop(2)

- dst = ctx.stack_push(Type::UnknownImm)

- asm.mov(dst, :rax)

-

- true

- elsif C.rb_class_of(comptime_a) == String && C.rb_class_of(comptime_b) == String

- unless Invariants.assume_bop_not_redefined(jit, C::STRING_REDEFINED_OP_FLAG, C::BOP_EQ)

- # if overridden, emit the generic version

- return false

- end

-

- # Guard that a is a String

- jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_a), a_opnd, StackOpnd[1], comptime_a, side_exit)

-

- equal_label = asm.new_label(:equal)

- ret_label = asm.new_label(:ret)

-

- # If they are equal by identity, return true

- asm.mov(:rax, a_opnd)

- asm.mov(:rcx, b_opnd)

- asm.cmp(:rax, :rcx)

- asm.je(equal_label)

-

- # Otherwise guard that b is a T_STRING (from type info) or String (from runtime guard)

- btype = ctx.get_opnd_type(StackOpnd[0])

- unless btype.string?

- # Note: any T_STRING is valid here, but we check for a ::String for simplicity

- # To pass a mutable static variable (rb_cString) requires an unsafe block

- jit_guard_known_klass(jit, ctx, asm, C.rb_class_of(comptime_b), b_opnd, StackOpnd[0], comptime_b, side_exit)

- end

-

- asm.comment('call rb_str_eql_internal')

- asm.mov(C_ARGS[0], a_opnd)

- asm.mov(C_ARGS[1], b_opnd)

- asm.call(gen_eq ? C.rb_str_eql_internal : C.rjit_str_neq_internal)

-

- # Push the output on the stack

- ctx.stack_pop(2)

- dst = ctx.stack_push(Type::UnknownImm)

- asm.mov(dst, C_RET)

- asm.jmp(ret_label)

-

- asm.write_label(equal_label)

- asm.mov(dst, gen_eq ? Qtrue : Qfalse)

-

- asm.write_label(ret_label)

-

- true

- else

- false

- end

- end

-

- # NOTE: This clobbers :rax

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_prepare_routine_call(jit, ctx, asm)

- jit.record_boundary_patch_point = true

- jit_save_pc(jit, asm)

- jit_save_sp(ctx, asm)

-

- # In case the routine calls Ruby methods, it can set local variables

- # through Kernel#binding and other means.

- ctx.clear_local_types

- end

-

- # NOTE: This clobbers :rax

- # @param jit [RubyVM::RJIT::JITState]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_save_pc(jit, asm, comment: 'save PC to CFP')

- next_pc = jit.pc + jit.insn.len * C.VALUE.size # Use the next one for backtrace and side exits

- asm.comment(comment)

- asm.mov(:rax, next_pc)

- asm.mov([CFP, C.rb_control_frame_t.offsetof(:pc)], :rax)

- end

-

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_save_sp(ctx, asm)

- if ctx.sp_offset != 0

- asm.comment('save SP to CFP')

- asm.lea(SP, ctx.sp_opnd)

- asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], SP)

- ctx.sp_offset = 0

- end

- end

-

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jump_to_next_insn(jit, ctx, asm)

- reset_depth = ctx.dup

- reset_depth.chain_depth = 0

-

- next_pc = jit.pc + jit.insn.len * C.VALUE.size

-

- # We are at the end of the current instruction. Record the boundary.

- if jit.record_boundary_patch_point

- exit_pos = Assembler.new.then do |ocb_asm|

- @exit_compiler.compile_side_exit(next_pc, ctx, ocb_asm)

- @ocb.write(ocb_asm)

- end

- Invariants.record_global_inval_patch(asm, exit_pos)

- jit.record_boundary_patch_point = false

- end

-

- jit_direct_jump(jit.iseq, next_pc, reset_depth, asm, comment: 'jump_to_next_insn')

- end

-

- # rb_vm_check_ints

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_check_ints(jit, ctx, asm)

- asm.comment('RUBY_VM_CHECK_INTS(ec)')

- asm.mov(:eax, DwordPtr[EC, C.rb_execution_context_t.offsetof(:interrupt_flag)])

- asm.test(:eax, :eax)

- asm.jnz(side_exit(jit, ctx))

- end

-

- # See get_lvar_level in compile.c

- def get_lvar_level(iseq)

- level = 0

- while iseq.to_i != iseq.body.local_iseq.to_i

- level += 1

- iseq = iseq.body.parent_iseq

- end

- return level

- end

-

- # GET_LEP

- # @param jit [RubyVM::RJIT::JITState]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_get_lep(jit, asm, reg:)

- level = get_lvar_level(jit.iseq)

- jit_get_ep(asm, level, reg:)

- end

-

- # vm_get_ep

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_get_ep(asm, level, reg:)

- asm.mov(reg, [CFP, C.rb_control_frame_t.offsetof(:ep)])

- level.times do

- # GET_PREV_EP: ep[VM_ENV_DATA_INDEX_SPECVAL] & ~0x03

- asm.mov(reg, [reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL])

- asm.and(reg, ~0x03)

- end

- end

-

- # vm_getivar

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_getivar(jit, ctx, asm, comptime_obj, ivar_id, obj_opnd, obj_yarv_opnd)

- side_exit = side_exit(jit, ctx)

- starting_ctx = ctx.dup # copy for jit_chain_guard

-

- # Guard not special const

- if C::SPECIAL_CONST_P(comptime_obj)

- asm.incr_counter(:getivar_special_const)

- return CantCompile

- end

-

- case C::BUILTIN_TYPE(comptime_obj)

- when C::T_OBJECT

- # This is the only supported case for now (ROBJECT_IVPTR)

- else

- # General case. Call rb_ivar_get().

- # VALUE rb_ivar_get(VALUE obj, ID id)

- asm.comment('call rb_ivar_get()')

- asm.mov(C_ARGS[0], obj_opnd ? obj_opnd : [CFP, C.rb_control_frame_t.offsetof(:self)])

- asm.mov(C_ARGS[1], ivar_id)

-

- # The function could raise exceptions.

- jit_prepare_routine_call(jit, ctx, asm) # clobbers obj_opnd and :rax

-

- asm.call(C.rb_ivar_get)

-

- if obj_opnd # attr_reader

- ctx.stack_pop

- end

-

- # Push the ivar on the stack

- out_opnd = ctx.stack_push(Type::Unknown)

- asm.mov(out_opnd, C_RET)

-

- # Jump to next instruction. This allows guard chains to share the same successor.

- jump_to_next_insn(jit, ctx, asm)

- return EndBlock

- end

-

- asm.mov(:rax, obj_opnd ? obj_opnd : [CFP, C.rb_control_frame_t.offsetof(:self)])

- guard_object_is_heap(jit, ctx, asm, :rax, obj_yarv_opnd, :getivar_not_heap)

-

- shape_id = C.rb_shape_get_shape_id(comptime_obj)

- if shape_id == C::OBJ_TOO_COMPLEX_SHAPE_ID

- asm.incr_counter(:getivar_too_complex)

- return CantCompile

- end

-

- asm.comment('guard shape')

- asm.cmp(DwordPtr[:rax, C.rb_shape_id_offset], shape_id)

- jit_chain_guard(:jne, jit, starting_ctx, asm, counted_exit(side_exit, :getivar_megamorphic))

-

- if obj_opnd

- ctx.stack_pop # pop receiver for attr_reader

- end

-

- index = C.rb_shape_get_iv_index(shape_id, ivar_id)

- # If there is no IVAR index, then the ivar was undefined

- # when we entered the compiler. That means we can just return

- # nil for this shape + iv name

- if index.nil?

- stack_opnd = ctx.stack_push(Type::Nil)

- val_opnd = Qnil

- else

- asm.comment('ROBJECT_IVPTR')

- if C::FL_TEST_RAW(comptime_obj, C::ROBJECT_EMBED)

- # Access embedded array

- asm.mov(:rax, [:rax, C.RObject.offsetof(:as, :ary) + (index * C.VALUE.size)])

- else

- # Pull out an ivar table on heap

- asm.mov(:rax, [:rax, C.RObject.offsetof(:as, :heap, :ivptr)])

- # Read the table

- asm.mov(:rax, [:rax, index * C.VALUE.size])

- end

- stack_opnd = ctx.stack_push(Type::Unknown)

- val_opnd = :rax

- end

- asm.mov(stack_opnd, val_opnd)

-

- # Let guard chains share the same successor

- jump_to_next_insn(jit, ctx, asm)

- EndBlock

- end

-

- def jit_write_iv(asm, comptime_receiver, recv_reg, temp_reg, ivar_index, set_value, needs_extension)

- # Compile time self is embedded and the ivar index lands within the object

- embed_test_result = C::FL_TEST_RAW(comptime_receiver, C::ROBJECT_EMBED) && !needs_extension

-

- if embed_test_result

- # Find the IV offset

- offs = C.RObject.offsetof(:as, :ary) + ivar_index * C.VALUE.size

-

- # Write the IV

- asm.comment('write IV')

- asm.mov(temp_reg, set_value)

- asm.mov([recv_reg, offs], temp_reg)

- else

- # Compile time value is *not* embedded.

-

- # Get a pointer to the extended table

- asm.mov(recv_reg, [recv_reg, C.RObject.offsetof(:as, :heap, :ivptr)])

-

- # Write the ivar in to the extended table

- asm.comment("write IV");

- asm.mov(temp_reg, set_value)

- asm.mov([recv_reg, C.VALUE.size * ivar_index], temp_reg)

- end

- end

-

- # vm_caller_setup_arg_block: Handle VM_CALL_ARGS_BLOCKARG cases.

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def guard_block_arg(jit, ctx, asm, calling)

- if calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0

- block_arg_type = ctx.get_opnd_type(StackOpnd[0])

- case block_arg_type

- in Type::Nil

- calling.block_handler = C::VM_BLOCK_HANDLER_NONE

- in Type::BlockParamProxy

- calling.block_handler = C.rb_block_param_proxy

- else

- asm.incr_counter(:send_block_arg)

- return CantCompile

- end

- end

- end

-

- # vm_search_method

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_search_method(jit, ctx, asm, mid, calling)

- assert_equal(true, jit.at_current_insn?)

-

- # Generate a side exit

- side_exit = side_exit(jit, ctx)

-

- # kw_splat is not supported yet

- if calling.flags & C::VM_CALL_KW_SPLAT != 0

- asm.incr_counter(:send_kw_splat)

- return CantCompile

- end

-

- # Get a compile-time receiver and its class

- recv_idx = calling.argc + (calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 ? 1 : 0) # blockarg is not popped yet

- recv_idx += calling.send_shift

- comptime_recv = jit.peek_at_stack(recv_idx)

- comptime_recv_klass = C.rb_class_of(comptime_recv)

-

- # Guard the receiver class (part of vm_search_method_fastpath)

- recv_opnd = ctx.stack_opnd(recv_idx)

- megamorphic_exit = counted_exit(side_exit, :send_klass_megamorphic)

- jit_guard_known_klass(jit, ctx, asm, comptime_recv_klass, recv_opnd, StackOpnd[recv_idx], comptime_recv, megamorphic_exit)

-

- # Do method lookup (vm_cc_cme(cc) != NULL)

- cme = C.rb_callable_method_entry(comptime_recv_klass, mid)

- if cme.nil?

- asm.incr_counter(:send_missing_cme)

- return CantCompile # We don't support vm_call_method_name

- end

-

- # Invalidate on redefinition (part of vm_search_method_fastpath)

- Invariants.assume_method_lookup_stable(jit, cme)

-

- return cme, comptime_recv_klass

- end

-

- # vm_call_general

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_call_general(jit, ctx, asm, mid, calling, cme, known_recv_class)

- jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class)

- end

-

- # vm_call_method

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- # @param send_shift [Integer] The number of shifts needed for VM_CALL_OPT_SEND

- def jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class)

- # The main check of vm_call_method before vm_call_method_each_type

- case C::METHOD_ENTRY_VISI(cme)

- in C::METHOD_VISI_PUBLIC

- # You can always call public methods

- in C::METHOD_VISI_PRIVATE

- # Allow only callsites without a receiver

- if calling.flags & C::VM_CALL_FCALL == 0

- asm.incr_counter(:send_private)

- return CantCompile

- end

- in C::METHOD_VISI_PROTECTED

- # If the method call is an FCALL, it is always valid

- if calling.flags & C::VM_CALL_FCALL == 0

- # otherwise we need an ancestry check to ensure the receiver is valid to be called as protected

- jit_protected_callee_ancestry_guard(asm, cme, side_exit(jit, ctx))

- end

- end

-

- # Get a compile-time receiver

- recv_idx = calling.argc + (calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 ? 1 : 0) # blockarg is not popped yet

- recv_idx += calling.send_shift

- comptime_recv = jit.peek_at_stack(recv_idx)

- recv_opnd = ctx.stack_opnd(recv_idx)

-

- jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class)

- end

-

- # Generate ancestry guard for protected callee.

- # Calls to protected callees only go through when self.is_a?(klass_that_defines_the_callee).

- def jit_protected_callee_ancestry_guard(asm, cme, side_exit)

- # See vm_call_method().

- def_class = cme.defined_class

- # Note: PC isn't written to current control frame as rb_is_kind_of() shouldn't raise.

- # VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass);

-

- asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:self)])

- asm.mov(C_ARGS[1], to_value(def_class))

- asm.call(C.rb_obj_is_kind_of)

- asm.test(C_RET, C_RET)

- asm.jz(counted_exit(side_exit, :send_protected_check_failed))

- end

-

- # vm_call_method_each_type

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class)

- case cme.def.type

- in C::VM_METHOD_TYPE_ISEQ

- iseq = def_iseq_ptr(cme.def)

- jit_call_iseq(jit, ctx, asm, cme, calling, iseq)

- in C::VM_METHOD_TYPE_NOTIMPLEMENTED

- asm.incr_counter(:send_notimplemented)

- return CantCompile

- in C::VM_METHOD_TYPE_CFUNC

- jit_call_cfunc(jit, ctx, asm, cme, calling, known_recv_class:)

- in C::VM_METHOD_TYPE_ATTRSET

- jit_call_attrset(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd)

- in C::VM_METHOD_TYPE_IVAR

- jit_call_ivar(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd)

- in C::VM_METHOD_TYPE_MISSING

- asm.incr_counter(:send_missing)

- return CantCompile

- in C::VM_METHOD_TYPE_BMETHOD

- jit_call_bmethod(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class)

- in C::VM_METHOD_TYPE_ALIAS

- jit_call_alias(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class)

- in C::VM_METHOD_TYPE_OPTIMIZED

- jit_call_optimized(jit, ctx, asm, cme, calling, known_recv_class)

- in C::VM_METHOD_TYPE_UNDEF

- asm.incr_counter(:send_undef)

- return CantCompile

- in C::VM_METHOD_TYPE_ZSUPER

- asm.incr_counter(:send_zsuper)

- return CantCompile

- in C::VM_METHOD_TYPE_REFINED

- asm.incr_counter(:send_refined)

- return CantCompile

- end

- end

-

- # vm_call_iseq_setup

- # @param jit [RubyVM::RJIT::JITState]

- # @param ctx [RubyVM::RJIT::Context]

- # @param asm [RubyVM::RJIT::Assembler]

- def jit_call_iseq(jit, ctx, asm, cme, calling, iseq, frame_type: nil, prev_ep: nil)

- argc = calling.argc

- flags = calling.flags

- send_shift = calling.send_shift

-

- # When you have keyword arguments, there is an extra object that gets

- # placed on the stack the represents a bitmap of the keywords that were not

- # specified at the call site. We need to keep track of the fact that this

- # value is present on the stack in order to properly set up the callee's

- # stack pointer.

- doing_kw_call = iseq.body.param.flags.has_kw

- supplying_kws = flags & C::VM_CALL_KWARG != 0

-

- if flags & C::VM_CALL_TAILCALL != 0

- # We can't handle tailcalls

- asm.incr_counter(:send_tailcall)

- return CantCompile

- end

-

- # No support for callees with these parameters yet as they require allocation

- # or complex handling.

- if iseq.body.param.flags.has_post

- asm.incr_counter(:send_iseq_has_opt)

- return CantCompile

- end

- if iseq.body.param.flags.has_kwrest

- asm.incr_counter(:send_iseq_has_kwrest)

- return CantCompile

- end

-

- # In order to handle backwards compatibility between ruby 3 and 2

- # ruby2_keywords was introduced. It is called only on methods

- # with splat and changes they way they handle them.

- # We are just going to not compile these.

- # https://www.rubydoc.info/stdlib/core/Proc:ruby2_keywords

- if iseq.body.param.flags.ruby2_keywords && flags & C::VM_CALL_ARGS_SPLAT != 0

- asm.incr_counter(:send_iseq_ruby2_keywords)

- return CantCompile

- end

-

- iseq_has_rest = iseq.body.param.flags.has_rest

- if iseq_has_rest && calling.block_handler == :captured

- asm.incr_counter(:send_iseq_has_rest_and_captured)

- return CantCompile

- end

-

- if iseq_has_rest && iseq.body.param.flags.has_kw && supplying_kws

- asm.incr_counter(:send_iseq_has_rest_and_kw_supplied)

- return CantCompile

- end

-

- # If we have keyword arguments being passed to a callee that only takes

- # positionals, then we need to allocate a hash. For now we're going to

- # call that too complex and bail.

- if supplying_kws && !iseq.body.param.flags.has_kw

- asm.incr_counter(:send_iseq_has_no_kw)

- return CantCompile

- end

-

- # If we have a method accepting no kwargs (**nil), exit if we have passed

- # it any kwargs.

- if supplying_kws && iseq.body.param.flags.accepts_no_kwarg

- asm.incr_counter(:send_iseq_accepts_no_kwarg)

- return CantCompile

- end

-

- # For computing number of locals to set up for the callee

- num_params = iseq.body.param.size

-

- # Block parameter handling. This mirrors setup_parameters_complex().

- if iseq.body.param.flags.has_block

- if iseq.body.local_iseq.to_i == iseq.to_i

- num_params -= 1

- else

- # In this case (param.flags.has_block && local_iseq != iseq),

- # the block argument is setup as a local variable and requires

- # materialization (allocation). Bail.

- asm.incr_counter(:send_iseq_materialized_block)

- return CantCompile

- end

- end

-

- if flags & C::VM_CALL_ARGS_SPLAT != 0 && flags & C::VM_CALL_ZSUPER != 0

- # zsuper methods are super calls without any arguments.

- # They are also marked as splat, but don't actually have an array

- # they pull arguments from, instead we need to change to call

- # a different method with the current stack.

- asm.incr_counter(:send_iseq_zsuper)

- return CantCompile

- end

-

- start_pc_offset = 0

- required_num = iseq.body.param.lead_num

-

- # This struct represents the metadata about the caller-specified

- # keyword arguments.

- kw_arg = calling.kwarg

- kw_arg_num = if kw_arg.nil?

- 0

- else

- kw_arg.keyword_len

- end

-

- # Arity handling and optional parameter setup

- opts_filled = argc - required_num - kw_arg_num

- opt_num = iseq.body.param.opt_num

- opts_missing = opt_num - opts_filled

-

- if doing_kw_call && flags & C::VM_CALL_ARGS_SPLAT != 0

- asm.incr_counter(:send_iseq_splat_with_kw)

- return CantCompile

- end

-

- if flags & C::VM_CALL_KW_SPLAT != 0

- asm.incr_counter(:send_iseq_kw_splat)

- return CantCompile

- end

-

- if iseq_has_rest && opt_num != 0

- asm.incr_counter(:send_iseq_has_rest_and_optional)

- return CantCompile

- end

-

- if opts_filled < 0 && flags & C::VM_CALL_ARGS_SPLAT == 0

- # Too few arguments and no splat to make up for it

- asm.incr_counter(:send_iseq_arity_error)

- return CantCompile

- end

-

- if opts_filled > opt_num && !iseq_has_rest

- # Too many arguments and no place to put them (i.e. rest arg)

- asm.incr_counter(:send_iseq_arity_error)

- return CantCompile

- end

-

- block_arg = flags & C::VM_CALL_ARGS_BLOCKARG != 0

-

- # Guard block_arg_type

- if guard_block_arg(jit, ctx, asm, calling) == CantCompile

- return CantCompile

- end

-

- # If we have unfilled optional arguments and keyword arguments then we

- # would need to adjust the arguments location to account for that.

- # For now we aren't handling this case.

- if doing_kw_call && opts_missing > 0

- asm.incr_counter(:send_iseq_missing_optional_kw)

- return CantCompile

- end

-

- # We will handle splat case later

- if opt_num > 0 && flags & C::VM_CALL_ARGS_SPLAT == 0

- num_params -= opts_missing

- start_pc_offset = iseq.body.param.opt_table[opts_filled]

- end

-

- if doing_kw_call

- # Here we're calling a method with keyword arguments and specifying

- # keyword arguments at this call site.

-

- # This struct represents the metadata about the callee-specified

- # keyword parameters.

- keyword = iseq.body.param.keyword

- keyword_num = keyword.num

- keyword_required_num = keyword.required_num

-

- required_kwargs_filled = 0

-

- if keyword_num > 30

- # We have so many keywords that (1 << num) encoded as a FIXNUM

- # (which shifts it left one more) no longer fits inside a 32-bit

- # immediate.

- asm.incr_counter(:send_iseq_too_many_kwargs)

- return CantCompile

- end

-

- # Check that the kwargs being passed are valid

- if supplying_kws

- # This is the list of keyword arguments that the callee specified

- # in its initial declaration.

- # SAFETY: see compile.c for sizing of this slice.

- callee_kwargs = keyword_num.times.map { |i| keyword.table[i] }

-

- # Here we're going to build up a list of the IDs that correspond to

- # the caller-specified keyword arguments. If they're not in the

- # same order as the order specified in the callee declaration, then

- # we're going to need to generate some code to swap values around

- # on the stack.

- caller_kwargs = []

- kw_arg.keyword_len.times do |kwarg_idx|

- sym = C.to_ruby(kw_arg[:keywords][kwarg_idx])

- caller_kwargs << C.rb_sym2id(sym)

- end

-

- # First, we're going to be sure that the names of every

- # caller-specified keyword argument correspond to a name in the

- # list of callee-specified keyword parameters.

- caller_kwargs.each do |caller_kwarg|

- search_result = callee_kwargs.map.with_index.find { |kwarg, _| kwarg == caller_kwarg }

-

- case search_result

- in nil

- # If the keyword was never found, then we know we have a

- # mismatch in the names of the keyword arguments, so we need to

- # bail.

- asm.incr_counter(:send_iseq_kwargs_mismatch)

- return CantCompile

- in _, callee_idx if callee_idx < keyword_required_num

- # Keep a count to ensure all required kwargs are specified

- required_kwargs_filled += 1

- else

- end

- end

- end

- assert_equal(true, required_kwargs_filled <= keyword_required_num)

- if required_kwargs_filled != keyword_required_num

- asm.incr_counter(:send_iseq_kwargs_mismatch)

- return CantCompile

- end

- end

-

- # Check if we need the arg0 splat handling of vm_callee_setup_block_arg

- arg_setup_block = (calling.block_handler == :captured) # arg_setup_type: arg_setup_block (invokeblock)

- block_arg0_splat = arg_setup_block && argc == 1 &&

- (iseq.body.param.flags.has_lead || opt_num > 1) &&

- !iseq.body.param.flags.ambiguous_param0

- if block_arg0_splat

- # If block_arg0_splat, we still need side exits after splat, but

- # doing push_splat_args here disallows it. So bail out.

- if flags & C::VM_CALL_ARGS_SPLAT != 0 && !iseq_has_rest

- asm.incr_counter(:invokeblock_iseq_arg0_args_splat)

- return CantCompile

- end

- # The block_arg0_splat implementation is for the rb_simple_iseq_p case,

- # but doing_kw_call means it's not a simple ISEQ.

- if doing_kw_call

- asm.incr_counter(:invokeblock_iseq_arg0_has_kw)

- return CantCompile

- end

- # The block_arg0_splat implementation cannot deal with optional parameters.

- # This is a setup_parameters_complex() situation and interacts with the

- # starting position of the callee.

- if opt_num > 1

- asm.incr_counter(:invokeblock_iseq_arg0_optional)

- return CantCompile

- end

- end

- if flags & C::VM_CALL_ARGS_SPLAT != 0 && !iseq_has_rest

- array = jit.peek_at_stack(block_arg ? 1 : 0)

- splat_array_length = if array.nil?

- 0

- else

- array.length

- end

-

- if opt_num == 0 && required_num != splat_array_length + argc - 1

- asm.incr_counter(:send_iseq_splat_arity_error)

- return CantCompile

- end

- end

-

- # Don't compile forwardable iseqs

- if iseq.body.param.flags.forwardable

- return CantCompile

- end

-

- # We will not have CantCompile from here.

-

- if block_arg

- ctx.stack_pop(1)

- end

-

- if calling.block_handler == C::VM_BLOCK_HANDLER_NONE && iseq.body.builtin_attrs & C::BUILTIN_ATTR_LEAF != 0

- if jit_leaf_builtin_func(jit, ctx, asm, flags, iseq)

- return KeepCompiling

- end

- end

-

- # Number of locals that are not parameters

- num_locals = iseq.body.local_table_size - num_params

-

- # Stack overflow check

- # Note that vm_push_frame checks it against a decremented cfp, hence the multiply by 2.

- # #define CHECK_VM_STACK_OVERFLOW0(cfp, sp, margin)

- asm.comment('stack overflow check')

- locals_offs = C.VALUE.size * (num_locals + iseq.body.stack_max) + 2 * C.rb_control_frame_t.size

- asm.lea(:rax, ctx.sp_opnd(locals_offs))

- asm.cmp(CFP, :rax)

- asm.jbe(counted_exit(side_exit(jit, ctx), :send_stackoverflow))

-

- # push_splat_args does stack manipulation so we can no longer side exit

- if splat_array_length

- remaining_opt = (opt_num + required_num) - (splat_array_length + (argc - 1))

-

- if opt_num > 0

- # We are going to jump to the correct offset based on how many optional