ZJIT: Add NoSingletonClass patch point (#14680)

* ZJIT: Add NoSingletonClass patch point

This patch point makes sure that when the object has a singleton class,
the JIT code is invalidated. As of now, this is only needed for C call
optimization.

In YJIT, the singleton class guard only applies to Array, Hash, and String.
But in ZJIT, we may optimize C calls from gems (e.g. `sqlite3`). So the
patch point needs to be applied to a broader range of classes.

* ZJIT: Only generate NoSingletonClass guard when the type can have singleton class

* ZJIT: Update or forget NoSingletonClass patch point when needed
This commit is contained in:
Stan Lo 2025-10-02 17:03:25 +01:00 committed by GitHub
parent 81f253577a
commit 2ed5a02fcc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 346 additions and 136 deletions

View File

@ -31,6 +31,7 @@
#include "ruby/st.h"
#include "vm_core.h"
#include "yjit.h"
#include "zjit.h"
/* Flags of T_CLASS
*
@ -1309,6 +1310,7 @@ make_singleton_class(VALUE obj)
RBASIC_SET_CLASS(obj, klass);
rb_singleton_class_attached(klass, obj);
rb_yjit_invalidate_no_singleton_class(orig_class);
rb_zjit_invalidate_no_singleton_class(orig_class);
SET_METACLASS_OF(klass, METACLASS_OF(rb_class_real(orig_class)));
return klass;

1
depend
View File

@ -1170,6 +1170,7 @@ class.$(OBJEXT): {$(VPATH)}vm_debug.h
class.$(OBJEXT): {$(VPATH)}vm_opts.h
class.$(OBJEXT): {$(VPATH)}vm_sync.h
class.$(OBJEXT): {$(VPATH)}yjit.h
class.$(OBJEXT): {$(VPATH)}zjit.h
compar.$(OBJEXT): $(hdrdir)/ruby/ruby.h
compar.$(OBJEXT): $(hdrdir)/ruby/version.h
compar.$(OBJEXT): $(top_srcdir)/internal/basic_operators.h

3
gc.c
View File

@ -1311,6 +1311,9 @@ rb_gc_obj_free(void *objspace, VALUE obj)
break;
case T_MODULE:
case T_CLASS:
#if USE_ZJIT
rb_zjit_klass_free(obj);
#endif
args.klass = obj;
rb_class_classext_foreach(obj, classext_free, (void *)&args);
if (RCLASS_CLASSEXT_TBL(obj)) {

View File

@ -2912,6 +2912,65 @@ class TestZJIT < Test::Unit::TestCase
}, call_threshold: 2, insns: [:opt_new]
end
def test_singleton_class_invalidation_annotated_ccall
assert_compiles '[false, true]', %q{
def define_singleton(obj, define)
if define
# Wrap in C method frame to avoid exiting JIT on defineclass
[nil].reverse_each do
class << obj
def ==(_)
true
end
end
end
end
false
end
def test(define)
obj = BasicObject.new
# This == call gets compiled to a CCall
obj == define_singleton(obj, define)
end
result = []
result << test(false) # Compiles BasicObject#==
result << test(true) # Should use singleton#== now
result
}, call_threshold: 2
end
def test_singleton_class_invalidation_optimized_variadic_ccall
assert_compiles '[1, 1000]', %q{
def define_singleton(arr, define)
if define
# Wrap in C method frame to avoid exiting JIT on defineclass
[nil].reverse_each do
class << arr
def push(x)
super(x * 1000)
end
end
end
end
1
end
def test(define)
arr = []
val = define_singleton(arr, define)
arr.push(val) # This CCall should be invalidated if singleton was defined
arr[0]
end
result = []
result << test(false) # Compiles Array#push as CCall
result << test(true) # Singleton defined, CCall should be invalidated
result
}, call_threshold: 2
end
private
# Assert that every method call in `test_script` can be compiled by ZJIT

3
zjit.h
View File

@ -19,6 +19,7 @@ void rb_zjit_profile_enable(const rb_iseq_t *iseq);
void rb_zjit_bop_redefined(int redefined_flag, enum ruby_basic_operators bop);
void rb_zjit_cme_invalidate(const rb_callable_method_entry_t *cme);
void rb_zjit_cme_free(const rb_callable_method_entry_t *cme);
void rb_zjit_klass_free(VALUE klass);
void rb_zjit_invalidate_no_ep_escape(const rb_iseq_t *iseq);
void rb_zjit_constant_state_changed(ID id);
void rb_zjit_iseq_mark(void *payload);
@ -26,6 +27,7 @@ void rb_zjit_iseq_update_references(void *payload);
void rb_zjit_iseq_free(const rb_iseq_t *iseq);
void rb_zjit_before_ractor_spawn(void);
void rb_zjit_tracing_invalidate_all(void);
void rb_zjit_invalidate_no_singleton_class(VALUE klass);
#else
#define rb_zjit_enabled_p false
static inline void rb_zjit_compile_iseq(const rb_iseq_t *iseq, bool jit_exception) {}
@ -37,6 +39,7 @@ static inline void rb_zjit_invalidate_no_ep_escape(const rb_iseq_t *iseq) {}
static inline void rb_zjit_constant_state_changed(ID id) {}
static inline void rb_zjit_before_ractor_spawn(void) {}
static inline void rb_zjit_tracing_invalidate_all(void) {}
static inline void rb_zjit_invalidate_no_singleton_class(VALUE klass) {}
#endif // #if USE_ZJIT
#endif // #ifndef ZJIT_H

View File

@ -9,7 +9,10 @@ use std::slice;
use crate::asm::Label;
use crate::backend::current::{Reg, ALLOC_REGS};
use crate::invariants::{track_bop_assumption, track_cme_assumption, track_no_ep_escape_assumption, track_no_trace_point_assumption, track_single_ractor_assumption, track_stable_constant_names_assumption};
use crate::invariants::{
track_bop_assumption, track_cme_assumption, track_no_ep_escape_assumption, track_no_trace_point_assumption,
track_single_ractor_assumption, track_stable_constant_names_assumption, track_no_singleton_class_assumption
};
use crate::gc::{append_gc_offsets, get_or_create_iseq_payload, get_or_create_iseq_payload_ptr, IseqCodePtrs, IseqPayload, IseqStatus};
use crate::state::ZJITState;
use crate::stats::{send_fallback_counter, exit_counter_for_compile_error, incr_counter, incr_counter_by, send_fallback_counter_for_method_type, send_fallback_counter_ptr_for_opcode, CompileError};
@ -654,6 +657,9 @@ fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invarian
Invariant::SingleRactorMode => {
track_single_ractor_assumption(code_ptr, side_exit_ptr, payload_ptr);
}
Invariant::NoSingletonClass { klass } => {
track_no_singleton_class_assumption(klass, code_ptr, side_exit_ptr, payload_ptr);
}
}
});
}

View File

@ -450,6 +450,16 @@ impl VALUE {
self.static_sym_p() || self.dynamic_sym_p()
}
pub fn instance_can_have_singleton_class(self) -> bool {
if self == unsafe { rb_cInteger } || self == unsafe { rb_cFloat } ||
self == unsafe { rb_cSymbol } || self == unsafe { rb_cNilClass } ||
self == unsafe { rb_cTrueClass } || self == unsafe { rb_cFalseClass } {
return false
}
true
}
/// Return true for a static (non-heap) Ruby symbol (RB_STATIC_SYM_P)
pub fn static_sym_p(self) -> bool {
let VALUE(cval) = self;

View File

@ -148,6 +148,16 @@ pub extern "C" fn rb_zjit_cme_free(cme: *const rb_callable_method_entry_struct)
invariants.forget_cme(cme);
}
/// GC callback for finalizing a class
#[unsafe(no_mangle)]
pub extern "C" fn rb_zjit_klass_free(klass: VALUE) {
if !ZJITState::has_instance() {
return;
}
let invariants = ZJITState::get_invariants();
invariants.forget_klass(klass);
}
/// GC callback for updating object references after all object moves
#[unsafe(no_mangle)]
pub extern "C" fn rb_zjit_root_update_references() {

View File

@ -141,6 +141,11 @@ pub enum Invariant {
NoEPEscape(IseqPtr),
/// There is one ractor running. If a non-root ractor gets spawned, this is invalidated.
SingleRactorMode,
/// Objects of this class have no singleton class.
/// When a singleton class is created for an object of this class, this is invalidated.
NoSingletonClass {
klass: VALUE,
},
}
impl Invariant {
@ -255,6 +260,12 @@ impl<'a> std::fmt::Display for InvariantPrinter<'a> {
Invariant::NoTracePoint => write!(f, "NoTracePoint"),
Invariant::NoEPEscape(iseq) => write!(f, "NoEPEscape({})", &iseq_name(iseq)),
Invariant::SingleRactorMode => write!(f, "SingleRactorMode"),
Invariant::NoSingletonClass { klass } => {
let class_name = get_class_name(klass);
write!(f, "NoSingletonClass({}@{:p})",
class_name,
self.ptr_map.map_ptr(klass.as_ptr::<VALUE>()))
}
}
}
}
@ -2021,6 +2032,9 @@ impl Function {
self.push_insn_id(block, insn_id); continue;
}
self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state });
if klass.instance_can_have_singleton_class() {
self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoSingletonClass { klass }, state });
}
if let Some(profiled_type) = profiled_type {
recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state });
}
@ -2028,6 +2042,9 @@ impl Function {
self.make_equal_to(insn_id, send_direct);
} else if def_type == VM_METHOD_TYPE_IVAR && args.is_empty() {
self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state });
if klass.instance_can_have_singleton_class() {
self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoSingletonClass { klass }, state });
}
if let Some(profiled_type) = profiled_type {
recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state });
}
@ -2097,6 +2114,7 @@ impl Function {
};
if recv_type.is_string() {
self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoSingletonClass { klass: recv_type.class() }, state });
let guard = self.push_insn(block, Insn::GuardType { val, guard_type: types::String, state });
// Infer type so AnyToString can fold off this
self.insn_types[guard.0] = self.infer_type(guard);
@ -2224,6 +2242,11 @@ impl Function {
/// Optimize SendWithoutBlock that land in a C method to a direct CCall without
/// runtime lookup.
fn optimize_c_calls(&mut self) {
fn gen_patch_points_for_optimized_ccall(fun: &mut Function, block: BlockId, recv_class: VALUE, method_id: ID, method: *const rb_callable_method_entry_struct, state: InsnId) {
fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoTracePoint, state });
fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass: recv_class, method: method_id, cme: method }, state });
}
// Try to reduce one SendWithoutBlock to a CCall
fn reduce_to_ccall(
fun: &mut Function,
@ -2285,12 +2308,16 @@ impl Function {
let ci_flags = unsafe { vm_ci_flag(call_info) };
// Filter for simple call sites (i.e. no splats etc.)
if ci_flags & VM_CALL_ARGS_SIMPLE != 0 {
// Commit to the replacement. Put PatchPoint.
fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass: recv_class, method: method_id, cme: method }, state });
gen_patch_points_for_optimized_ccall(fun, block, recv_class, method_id, method, state);
if recv_class.instance_can_have_singleton_class() {
fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoSingletonClass { klass: recv_class }, state });
}
if let Some(profiled_type) = profiled_type {
// Guard receiver class
recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state });
}
let cfun = unsafe { get_mct_func(cfunc) }.cast();
let mut cfunc_args = vec![recv];
cfunc_args.append(&mut args);
@ -2308,16 +2335,11 @@ impl Function {
// func(int argc, VALUE *argv, VALUE recv)
let ci_flags = unsafe { vm_ci_flag(call_info) };
if ci_flags & VM_CALL_ARGS_SIMPLE != 0 {
fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoTracePoint, state });
fun.push_insn(block, Insn::PatchPoint {
invariant: Invariant::MethodRedefined {
klass: recv_class,
method: method_id,
cme: method
},
state
});
gen_patch_points_for_optimized_ccall(fun, block, recv_class, method_id, method, state);
if recv_class.instance_can_have_singleton_class() {
fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoSingletonClass { klass: recv_class }, state });
}
if let Some(profiled_type) = profiled_type {
// Guard receiver class
recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state });
@ -8448,10 +8470,11 @@ mod opt_tests {
Jump bb2(v4)
bb2(v6:BasicObject):
PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010)
v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v19:BasicObject = SendWithoutBlockDirect v18, :foo (0x1038)
PatchPoint NoSingletonClass(Object@0x1000)
v19:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v20:BasicObject = SendWithoutBlockDirect v19, :foo (0x1038)
CheckInterrupts
Return v19
Return v20
");
}
@ -8504,10 +8527,11 @@ mod opt_tests {
Jump bb2(v4)
bb2(v6:BasicObject):
PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010)
v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v19:BasicObject = SendWithoutBlockDirect v18, :foo (0x1038)
PatchPoint NoSingletonClass(Object@0x1000)
v19:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v20:BasicObject = SendWithoutBlockDirect v19, :foo (0x1038)
CheckInterrupts
Return v19
Return v20
");
}
@ -8531,10 +8555,11 @@ mod opt_tests {
bb2(v6:BasicObject):
v10:Fixnum[3] = Const Value(3)
PatchPoint MethodRedefined(Object@0x1000, Integer@0x1008, cme:0x1010)
v19:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v20:BasicObject = SendWithoutBlockDirect v19, :Integer (0x1038), v10
PatchPoint NoSingletonClass(Object@0x1000)
v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v21:BasicObject = SendWithoutBlockDirect v20, :Integer (0x1038), v10
CheckInterrupts
Return v20
Return v21
");
}
@ -8561,10 +8586,11 @@ mod opt_tests {
v10:Fixnum[1] = Const Value(1)
v11:Fixnum[2] = Const Value(2)
PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010)
v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v21:BasicObject = SendWithoutBlockDirect v20, :foo (0x1038), v10, v11
PatchPoint NoSingletonClass(Object@0x1000)
v21:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v22:BasicObject = SendWithoutBlockDirect v21, :foo (0x1038), v10, v11
CheckInterrupts
Return v21
Return v22
");
}
@ -8592,13 +8618,15 @@ mod opt_tests {
Jump bb2(v4)
bb2(v6:BasicObject):
PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010)
v22:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v23:BasicObject = SendWithoutBlockDirect v22, :foo (0x1038)
PatchPoint NoSingletonClass(Object@0x1000)
v23:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v24:BasicObject = SendWithoutBlockDirect v23, :foo (0x1038)
PatchPoint MethodRedefined(Object@0x1000, bar@0x1040, cme:0x1048)
v25:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v26:BasicObject = SendWithoutBlockDirect v25, :bar (0x1038)
PatchPoint NoSingletonClass(Object@0x1000)
v27:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v28:BasicObject = SendWithoutBlockDirect v27, :bar (0x1038)
CheckInterrupts
Return v26
Return v28
");
}
@ -8623,10 +8651,11 @@ mod opt_tests {
v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000))
v12:StringExact = StringCopy v10
PatchPoint MethodRedefined(Object@0x1008, puts@0x1010, cme:0x1018)
v22:HeapObject[class_exact*:Object@VALUE(0x1008)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1008)]
v23:BasicObject = CCallVariadic puts@0x1040, v22, v12
PatchPoint NoSingletonClass(Object@0x1008)
v23:HeapObject[class_exact*:Object@VALUE(0x1008)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1008)]
v24:BasicObject = CCallVariadic puts@0x1040, v23, v12
CheckInterrupts
Return v23
Return v24
");
}
@ -9652,10 +9681,10 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(Integer@0x1000, itself@0x1008, cme:0x1010)
v21:Fixnum = GuardType v9, Fixnum
v22:BasicObject = CCall itself@0x1038, v21
v22:Fixnum = GuardType v9, Fixnum
v23:BasicObject = CCall itself@0x1038, v22
CheckInterrupts
Return v22
Return v23
");
}
@ -9676,9 +9705,10 @@ mod opt_tests {
bb2(v6:BasicObject):
v11:ArrayExact = NewArray
PatchPoint MethodRedefined(Array@0x1000, itself@0x1008, cme:0x1010)
v20:BasicObject = CCall itself@0x1038, v11
PatchPoint NoSingletonClass(Array@0x1000)
v22:BasicObject = CCall itself@0x1038, v11
CheckInterrupts
Return v20
Return v22
");
}
@ -9704,7 +9734,8 @@ mod opt_tests {
bb2(v8:BasicObject, v9:NilClass):
v14:ArrayExact = NewArray
PatchPoint MethodRedefined(Array@0x1000, itself@0x1008, cme:0x1010)
v28:BasicObject = CCall itself@0x1038, v14
PatchPoint NoSingletonClass(Array@0x1000)
v30:BasicObject = CCall itself@0x1038, v14
PatchPoint NoEPEscape(test)
v21:Fixnum[1] = Const Value(1)
CheckInterrupts
@ -9738,7 +9769,8 @@ mod opt_tests {
PatchPoint StableConstantNames(0x1000, M)
v29:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008))
PatchPoint MethodRedefined(Module@0x1010, name@0x1018, cme:0x1020)
v31:StringExact|NilClass = CCall name@0x1048, v29
PatchPoint NoSingletonClass(Module@0x1010)
v33:StringExact|NilClass = CCall name@0x1048, v29
PatchPoint NoEPEscape(test)
v21:Fixnum[1] = Const Value(1)
CheckInterrupts
@ -9768,7 +9800,8 @@ mod opt_tests {
bb2(v8:BasicObject, v9:NilClass):
v14:ArrayExact = NewArray
PatchPoint MethodRedefined(Array@0x1000, length@0x1008, cme:0x1010)
v28:Fixnum = CCall length@0x1038, v14
PatchPoint NoSingletonClass(Array@0x1000)
v30:Fixnum = CCall length@0x1038, v14
v21:Fixnum[5] = Const Value(5)
CheckInterrupts
Return v21
@ -9910,7 +9943,8 @@ mod opt_tests {
bb2(v8:BasicObject, v9:NilClass):
v14:ArrayExact = NewArray
PatchPoint MethodRedefined(Array@0x1000, size@0x1008, cme:0x1010)
v28:Fixnum = CCall size@0x1038, v14
PatchPoint NoSingletonClass(Array@0x1000)
v30:Fixnum = CCall size@0x1038, v14
v21:Fixnum[5] = Const Value(5)
CheckInterrupts
Return v21
@ -9991,9 +10025,10 @@ mod opt_tests {
v16:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000))
v18:ArrayExact = ArrayDup v16
PatchPoint MethodRedefined(Array@0x1008, first@0x1010, cme:0x1018)
v29:BasicObject = SendWithoutBlockDirect v18, :first (0x1040)
PatchPoint NoSingletonClass(Array@0x1008)
v30:BasicObject = SendWithoutBlockDirect v18, :first (0x1040)
CheckInterrupts
Return v29
Return v30
");
}
@ -10019,9 +10054,10 @@ mod opt_tests {
PatchPoint StableConstantNames(0x1000, M)
v21:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008))
PatchPoint MethodRedefined(Module@0x1010, class@0x1018, cme:0x1020)
v23:BasicObject = SendWithoutBlockDirect v21, :class (0x1048)
PatchPoint NoSingletonClass(Module@0x1010)
v24:BasicObject = SendWithoutBlockDirect v21, :class (0x1048)
CheckInterrupts
Return v23
Return v24
");
}
@ -10052,10 +10088,11 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010)
v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C]
v22:BasicObject = SendWithoutBlockDirect v21, :foo (0x1038)
PatchPoint NoSingletonClass(C@0x1000)
v22:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C]
v23:BasicObject = SendWithoutBlockDirect v22, :foo (0x1038)
CheckInterrupts
Return v22
Return v23
");
}
@ -10233,9 +10270,10 @@ mod opt_tests {
v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000))
v12:StringExact = StringCopy v10
PatchPoint MethodRedefined(String@0x1008, bytesize@0x1010, cme:0x1018)
v21:Fixnum = CCall bytesize@0x1040, v12
PatchPoint NoSingletonClass(String@0x1008)
v23:Fixnum = CCall bytesize@0x1040, v12
CheckInterrupts
Return v21
Return v23
");
}
@ -10361,7 +10399,8 @@ mod opt_tests {
PatchPoint MethodRedefined(C@0x1008, new@0x1010, cme:0x1018)
v43:HeapObject[class_exact:C] = ObjectAllocClass C:VALUE(0x1008)
PatchPoint MethodRedefined(C@0x1008, initialize@0x1040, cme:0x1048)
v45:NilClass = CCall initialize@0x1070, v43
PatchPoint NoSingletonClass(C@0x1008)
v47:NilClass = CCall initialize@0x1070, v43
CheckInterrupts
CheckInterrupts
Return v43
@ -10397,7 +10436,8 @@ mod opt_tests {
PatchPoint MethodRedefined(C@0x1008, new@0x1010, cme:0x1018)
v45:HeapObject[class_exact:C] = ObjectAllocClass C:VALUE(0x1008)
PatchPoint MethodRedefined(C@0x1008, initialize@0x1040, cme:0x1048)
v47:BasicObject = SendWithoutBlockDirect v45, :initialize (0x1070), v13
PatchPoint NoSingletonClass(C@0x1008)
v48:BasicObject = SendWithoutBlockDirect v45, :initialize (0x1070), v13
CheckInterrupts
CheckInterrupts
Return v45
@ -10427,7 +10467,8 @@ mod opt_tests {
PatchPoint MethodRedefined(Object@0x1008, new@0x1010, cme:0x1018)
v43:ObjectExact = ObjectAllocClass Object:VALUE(0x1008)
PatchPoint MethodRedefined(Object@0x1008, initialize@0x1040, cme:0x1048)
v45:NilClass = CCall initialize@0x1070, v43
PatchPoint NoSingletonClass(Object@0x1008)
v47:NilClass = CCall initialize@0x1070, v43
CheckInterrupts
CheckInterrupts
Return v43
@ -10457,7 +10498,8 @@ mod opt_tests {
PatchPoint MethodRedefined(BasicObject@0x1008, new@0x1010, cme:0x1018)
v43:BasicObjectExact = ObjectAllocClass BasicObject:VALUE(0x1008)
PatchPoint MethodRedefined(BasicObject@0x1008, initialize@0x1040, cme:0x1048)
v45:NilClass = CCall initialize@0x1070, v43
PatchPoint NoSingletonClass(BasicObject@0x1008)
v47:NilClass = CCall initialize@0x1070, v43
CheckInterrupts
CheckInterrupts
Return v43
@ -10517,9 +10559,10 @@ mod opt_tests {
v13:Fixnum[1] = Const Value(1)
PatchPoint MethodRedefined(Array@0x1008, new@0x1010, cme:0x1018)
PatchPoint MethodRedefined(Class@0x1040, new@0x1010, cme:0x1018)
v51:BasicObject = CCallVariadic new@0x1048, v42, v13
PatchPoint NoSingletonClass(Class@0x1040)
v53:BasicObject = CCallVariadic new@0x1048, v42, v13
CheckInterrupts
Return v51
Return v53
");
}
@ -10546,8 +10589,9 @@ mod opt_tests {
PatchPoint MethodRedefined(Set@0x1008, new@0x1010, cme:0x1018)
v16:HeapObject = ObjectAlloc v40
PatchPoint MethodRedefined(Set@0x1008, initialize@0x1040, cme:0x1048)
v45:SetExact = GuardType v16, SetExact
v46:BasicObject = CCallVariadic initialize@0x1070, v45
PatchPoint NoSingletonClass(Set@0x1008)
v46:SetExact = GuardType v16, SetExact
v47:BasicObject = CCallVariadic initialize@0x1070, v46
CheckInterrupts
CheckInterrupts
Return v16
@ -10576,9 +10620,10 @@ mod opt_tests {
v12:NilClass = Const Value(nil)
PatchPoint MethodRedefined(String@0x1008, new@0x1010, cme:0x1018)
PatchPoint MethodRedefined(Class@0x1040, new@0x1010, cme:0x1018)
v49:BasicObject = CCallVariadic new@0x1048, v40
PatchPoint NoSingletonClass(Class@0x1040)
v51:BasicObject = CCallVariadic new@0x1048, v40
CheckInterrupts
Return v49
Return v51
");
}
@ -10607,7 +10652,8 @@ mod opt_tests {
PatchPoint MethodRedefined(Regexp@0x1008, new@0x1018, cme:0x1020)
v47:RegexpExact = ObjectAllocClass Regexp:VALUE(0x1008)
PatchPoint MethodRedefined(Regexp@0x1008, initialize@0x1048, cme:0x1050)
v50:BasicObject = CCallVariadic initialize@0x1078, v47, v15
PatchPoint NoSingletonClass(Regexp@0x1008)
v51:BasicObject = CCallVariadic initialize@0x1078, v47, v15
CheckInterrupts
CheckInterrupts
Return v47
@ -10633,9 +10679,10 @@ mod opt_tests {
bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject):
v17:ArrayExact = NewArray v11, v12
PatchPoint MethodRedefined(Array@0x1000, length@0x1008, cme:0x1010)
v28:Fixnum = CCall length@0x1038, v17
PatchPoint NoSingletonClass(Array@0x1000)
v30:Fixnum = CCall length@0x1038, v17
CheckInterrupts
Return v28
Return v30
");
}
@ -10658,9 +10705,10 @@ mod opt_tests {
bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject):
v17:ArrayExact = NewArray v11, v12
PatchPoint MethodRedefined(Array@0x1000, size@0x1008, cme:0x1010)
v28:Fixnum = CCall size@0x1038, v17
PatchPoint NoSingletonClass(Array@0x1000)
v30:Fixnum = CCall size@0x1038, v17
CheckInterrupts
Return v28
Return v30
");
}
@ -11169,8 +11217,9 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000))
v25:String = GuardType v9, String
v19:StringExact = StringConcat v13, v25
PatchPoint NoSingletonClass(String@0x1008)
v26:String = GuardType v9, String
v19:StringExact = StringConcat v13, v26
CheckInterrupts
Return v19
");
@ -11200,8 +11249,9 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000))
v25:String = GuardType v9, String
v19:StringExact = StringConcat v13, v25
PatchPoint NoSingletonClass(MyString@0x1008)
v26:String = GuardType v9, String
v19:StringExact = StringConcat v13, v26
CheckInterrupts
Return v19
");
@ -11289,9 +11339,9 @@ mod opt_tests {
v13:Fixnum[1] = Const Value(1)
CheckInterrupts
PatchPoint MethodRedefined(Integer@0x1000, itself@0x1008, cme:0x1010)
v33:BasicObject = CCall itself@0x1038, v13
v34:BasicObject = CCall itself@0x1038, v13
CheckInterrupts
Return v33
Return v34
");
}
@ -11443,9 +11493,10 @@ mod opt_tests {
v10:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000))
v12:ArrayExact = ArrayDup v10
PatchPoint MethodRedefined(Array@0x1008, max@0x1010, cme:0x1018)
v21:BasicObject = SendWithoutBlockDirect v12, :max (0x1040)
PatchPoint NoSingletonClass(Array@0x1008)
v22:BasicObject = SendWithoutBlockDirect v12, :max (0x1040)
CheckInterrupts
Return v21
Return v22
");
}
@ -11515,9 +11566,9 @@ mod opt_tests {
bb2(v6:BasicObject):
v10:NilClass = Const Value(nil)
PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010)
v21:TrueClass = CCall nil?@0x1038, v10
v22:TrueClass = CCall nil?@0x1038, v10
CheckInterrupts
Return v21
Return v22
");
}
@ -11564,9 +11615,9 @@ mod opt_tests {
bb2(v6:BasicObject):
v10:Fixnum[1] = Const Value(1)
PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010)
v21:FalseClass = CCall nil?@0x1038, v10
v22:FalseClass = CCall nil?@0x1038, v10
CheckInterrupts
Return v21
Return v22
");
}
@ -11615,10 +11666,10 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010)
v23:NilClass = GuardType v9, NilClass
v24:TrueClass = CCall nil?@0x1038, v23
v24:NilClass = GuardType v9, NilClass
v25:TrueClass = CCall nil?@0x1038, v24
CheckInterrupts
Return v24
Return v25
");
}
@ -11641,10 +11692,10 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(FalseClass@0x1000, nil?@0x1008, cme:0x1010)
v23:FalseClass = GuardType v9, FalseClass
v24:FalseClass = CCall nil?@0x1038, v23
v24:FalseClass = GuardType v9, FalseClass
v25:FalseClass = CCall nil?@0x1038, v24
CheckInterrupts
Return v24
Return v25
");
}
@ -11667,10 +11718,10 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(TrueClass@0x1000, nil?@0x1008, cme:0x1010)
v23:TrueClass = GuardType v9, TrueClass
v24:FalseClass = CCall nil?@0x1038, v23
v24:TrueClass = GuardType v9, TrueClass
v25:FalseClass = CCall nil?@0x1038, v24
CheckInterrupts
Return v24
Return v25
");
}
@ -11693,10 +11744,10 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(Symbol@0x1000, nil?@0x1008, cme:0x1010)
v23:StaticSymbol = GuardType v9, StaticSymbol
v24:FalseClass = CCall nil?@0x1038, v23
v24:StaticSymbol = GuardType v9, StaticSymbol
v25:FalseClass = CCall nil?@0x1038, v24
CheckInterrupts
Return v24
Return v25
");
}
@ -11719,10 +11770,10 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010)
v23:Fixnum = GuardType v9, Fixnum
v24:FalseClass = CCall nil?@0x1038, v23
v24:Fixnum = GuardType v9, Fixnum
v25:FalseClass = CCall nil?@0x1038, v24
CheckInterrupts
Return v24
Return v25
");
}
@ -11745,10 +11796,10 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(Float@0x1000, nil?@0x1008, cme:0x1010)
v23:Flonum = GuardType v9, Flonum
v24:FalseClass = CCall nil?@0x1038, v23
v24:Flonum = GuardType v9, Flonum
v25:FalseClass = CCall nil?@0x1038, v24
CheckInterrupts
Return v24
Return v25
");
}
@ -11771,10 +11822,11 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(String@0x1000, nil?@0x1008, cme:0x1010)
v23:StringExact = GuardType v9, StringExact
v24:FalseClass = CCall nil?@0x1038, v23
PatchPoint NoSingletonClass(String@0x1000)
v25:StringExact = GuardType v9, StringExact
v26:FalseClass = CCall nil?@0x1038, v25
CheckInterrupts
Return v24
Return v26
");
}
@ -11797,10 +11849,11 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(Array@0x1000, !@0x1008, cme:0x1010)
v23:ArrayExact = GuardType v9, ArrayExact
v24:BoolExact = CCall !@0x1038, v23
PatchPoint NoSingletonClass(Array@0x1000)
v25:ArrayExact = GuardType v9, ArrayExact
v26:BoolExact = CCall !@0x1038, v25
CheckInterrupts
Return v24
Return v26
");
}
@ -11823,10 +11876,11 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(Array@0x1000, empty?@0x1008, cme:0x1010)
v23:ArrayExact = GuardType v9, ArrayExact
v24:BoolExact = CCall empty?@0x1038, v23
PatchPoint NoSingletonClass(Array@0x1000)
v25:ArrayExact = GuardType v9, ArrayExact
v26:BoolExact = CCall empty?@0x1038, v25
CheckInterrupts
Return v24
Return v26
");
}
@ -11849,10 +11903,11 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(Hash@0x1000, empty?@0x1008, cme:0x1010)
v23:HashExact = GuardType v9, HashExact
v24:BoolExact = CCall empty?@0x1038, v23
PatchPoint NoSingletonClass(Hash@0x1000)
v25:HashExact = GuardType v9, HashExact
v26:BoolExact = CCall empty?@0x1038, v25
CheckInterrupts
Return v24
Return v26
");
}
@ -11877,10 +11932,11 @@ mod opt_tests {
Jump bb2(v6, v7, v8)
bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject):
PatchPoint MethodRedefined(C@0x1000, ==@0x1008, cme:0x1010)
v26:HeapObject[class_exact:C] = GuardType v11, HeapObject[class_exact:C]
v27:BoolExact = CCall ==@0x1038, v26, v12
PatchPoint NoSingletonClass(C@0x1000)
v28:HeapObject[class_exact:C] = GuardType v11, HeapObject[class_exact:C]
v29:BoolExact = CCall ==@0x1038, v28, v12
CheckInterrupts
Return v27
Return v29
");
}
@ -11960,10 +12016,11 @@ mod opt_tests {
Jump bb2(v4)
bb2(v6:BasicObject):
PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010)
v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v19:BasicObject = SendWithoutBlockDirect v18, :foo (0x1038)
PatchPoint NoSingletonClass(Object@0x1000)
v19:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)]
v20:BasicObject = SendWithoutBlockDirect v19, :foo (0x1038)
CheckInterrupts
Return v19
Return v20
");
}
@ -11994,11 +12051,12 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010)
v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C]
v24:HeapObject[class_exact:C] = GuardShape v21, 0x1038
v25:BasicObject = LoadIvarEmbedded v24, :@foo@0x1039
PatchPoint NoSingletonClass(C@0x1000)
v22:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C]
v25:HeapObject[class_exact:C] = GuardShape v22, 0x1038
v26:BasicObject = LoadIvarEmbedded v25, :@foo@0x1039
CheckInterrupts
Return v25
Return v26
");
}
@ -12029,11 +12087,12 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010)
v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C]
v24:HeapObject[class_exact:C] = GuardShape v21, 0x1038
v25:BasicObject = LoadIvarEmbedded v24, :@foo@0x1039
PatchPoint NoSingletonClass(C@0x1000)
v22:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C]
v25:HeapObject[class_exact:C] = GuardShape v22, 0x1038
v26:BasicObject = LoadIvarEmbedded v25, :@foo@0x1039
CheckInterrupts
Return v25
Return v26
");
}
@ -12106,10 +12165,11 @@ mod opt_tests {
PatchPoint StableConstantNames(0x1000, O)
v21:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008))
PatchPoint MethodRedefined(C@0x1010, foo@0x1018, cme:0x1020)
v25:HeapObject[VALUE(0x1008)] = GuardShape v21, 0x1048
v26:NilClass = Const Value(nil)
PatchPoint NoSingletonClass(C@0x1010)
v26:HeapObject[VALUE(0x1008)] = GuardShape v21, 0x1048
v27:NilClass = Const Value(nil)
CheckInterrupts
Return v26
Return v27
");
}
@ -12139,10 +12199,11 @@ mod opt_tests {
PatchPoint StableConstantNames(0x1000, O)
v21:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008))
PatchPoint MethodRedefined(C@0x1010, foo@0x1018, cme:0x1020)
v25:HeapObject[VALUE(0x1008)] = GuardShape v21, 0x1048
v26:NilClass = Const Value(nil)
PatchPoint NoSingletonClass(C@0x1010)
v26:HeapObject[VALUE(0x1008)] = GuardShape v21, 0x1048
v27:NilClass = Const Value(nil)
CheckInterrupts
Return v26
Return v27
");
}
@ -12169,11 +12230,12 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010)
v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C]
v24:HeapObject[class_exact:C] = GuardShape v21, 0x1038
v25:NilClass = Const Value(nil)
PatchPoint NoSingletonClass(C@0x1000)
v22:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C]
v25:HeapObject[class_exact:C] = GuardShape v22, 0x1038
v26:NilClass = Const Value(nil)
CheckInterrupts
Return v25
Return v26
");
}
@ -12200,11 +12262,12 @@ mod opt_tests {
Jump bb2(v5, v6)
bb2(v8:BasicObject, v9:BasicObject):
PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010)
v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C]
v24:HeapObject[class_exact:C] = GuardShape v21, 0x1038
v25:NilClass = Const Value(nil)
PatchPoint NoSingletonClass(C@0x1000)
v22:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C]
v25:HeapObject[class_exact:C] = GuardShape v22, 0x1038
v26:NilClass = Const Value(nil)
CheckInterrupts
Return v25
Return v26
");
}

View File

@ -2,7 +2,7 @@
use std::{collections::{HashMap, HashSet}, mem};
use crate::{backend::lir::{asm_comment, Assembler}, cruby::{iseq_name, rb_callable_method_entry_t, rb_gc_location, ruby_basic_operators, src_loc, with_vm_lock, IseqPtr, RedefinitionFlag, ID}, gc::IseqPayload, hir::Invariant, options::debug, state::{zjit_enabled_p, ZJITState}, virtualmem::CodePtr};
use crate::{backend::lir::{asm_comment, Assembler}, cruby::{iseq_name, rb_callable_method_entry_t, rb_gc_location, ruby_basic_operators, src_loc, with_vm_lock, IseqPtr, RedefinitionFlag, ID, VALUE}, gc::IseqPayload, hir::Invariant, options::debug, state::{zjit_enabled_p, ZJITState}, virtualmem::CodePtr};
use crate::stats::with_time_stat;
use crate::stats::Counter::invalidation_time_ns;
use crate::gc::remove_gc_offsets;
@ -59,6 +59,10 @@ pub struct Invariants {
/// Set of patch points that assume that the interpreter is running with only one ractor
single_ractor_patch_points: HashSet<PatchPoint>,
/// Map from a class to a set of patch points that assume objects of the class
/// will have no singleton class.
no_singleton_class_patch_points: HashMap<VALUE, HashSet<PatchPoint>>,
}
impl Invariants {
@ -67,6 +71,7 @@ impl Invariants {
self.update_ep_escape_iseqs();
self.update_no_ep_escape_iseq_patch_points();
self.update_cme_patch_points();
self.update_no_singleton_class_patch_points();
}
/// Forget an ISEQ when freeing it. We need to because a) if the address is reused, we'd be
@ -85,6 +90,11 @@ impl Invariants {
self.cme_patch_points.remove(&cme);
}
/// Forget a class when freeing it. See [Self::forget_iseq] for reasoning.
pub fn forget_klass(&mut self, klass: VALUE) {
self.no_singleton_class_patch_points.remove(&klass);
}
/// Update ISEQ references in Invariants::ep_escape_iseqs
fn update_ep_escape_iseqs(&mut self) {
let updated = std::mem::take(&mut self.ep_escape_iseqs)
@ -116,6 +126,17 @@ impl Invariants {
.collect();
self.cme_patch_points = updated_cme_patch_points;
}
fn update_no_singleton_class_patch_points(&mut self) {
let updated_no_singleton_class_patch_points = std::mem::take(&mut self.no_singleton_class_patch_points)
.into_iter()
.map(|(klass, patch_points)| {
let new_klass = unsafe { rb_gc_location(klass) };
(new_klass, patch_points)
})
.collect();
self.no_singleton_class_patch_points = updated_no_singleton_class_patch_points;
}
}
/// Called when a basic operator is redefined. Note that all the blocks assuming
@ -245,6 +266,21 @@ pub fn track_stable_constant_names_assumption(
}
}
/// Track a patch point for objects of a given class will have no singleton class.
pub fn track_no_singleton_class_assumption(
klass: VALUE,
patch_point_ptr: CodePtr,
side_exit_ptr: CodePtr,
payload_ptr: *mut IseqPayload,
) {
let invariants = ZJITState::get_invariants();
invariants.no_singleton_class_patch_points.entry(klass).or_default().insert(PatchPoint {
patch_point_ptr,
side_exit_ptr,
payload_ptr,
});
}
/// Called when a method is redefined. Invalidates all JIT code that depends on the CME.
#[unsafe(no_mangle)]
pub extern "C" fn rb_zjit_cme_invalidate(cme: *const rb_callable_method_entry_t) {
@ -357,3 +393,20 @@ pub extern "C" fn rb_zjit_tracing_invalidate_all() {
cb.mark_all_executable();
});
}
#[unsafe(no_mangle)]
pub extern "C" fn rb_zjit_invalidate_no_singleton_class(klass: VALUE) {
if !zjit_enabled_p() {
return;
}
with_vm_lock(src_loc!(), || {
let invariants = ZJITState::get_invariants();
if let Some(patch_points) = invariants.no_singleton_class_patch_points.remove(&klass) {
let cb = ZJITState::get_code_block();
debug!("Singleton class created for {:?}", klass);
compile_patch_points!(cb, patch_points, "Singleton class created for {:?}", klass);
cb.mark_all_executable();
}
});
}