diff options
| -rw-r--r-- | class.c | 1 | ||||
| -rw-r--r-- | gc.c | 134 | ||||
| -rw-r--r-- | include/ruby/internal/core/rtypeddata.h | 5 | ||||
| -rw-r--r-- | include/ruby/internal/fl_type.h | 51 | ||||
| -rw-r--r-- | proc.c | 2 | ||||
| -rw-r--r-- | yjit/src/cruby_bindings.inc.rs | 5 |
6 files changed, 86 insertions, 112 deletions
@@ -232,7 +232,6 @@ class_alloc(VALUE flags, VALUE klass) size_t alloc_size = sizeof(struct RClass) + sizeof(rb_classext_t); flags &= T_MASK; - flags |= FL_PROMOTED1 /* start from age == 2 */; if (RGENGC_WB_PROTECTED_CLASS) flags |= FL_WB_PROTECTED; NEWOBJ_OF(obj, struct RClass, klass, flags, alloc_size, 0); @@ -939,6 +939,9 @@ static const bool HEAP_PAGE_ALLOC_USE_MMAP = false; static bool heap_page_alloc_use_mmap; #endif +#define RVALUE_AGE_BIT_COUNT 2 +#define RVALUE_AGE_BIT_MASK (((bits_t)1 << RVALUE_AGE_BIT_COUNT) - 1) + struct heap_page { short slot_size; short total_slots; @@ -968,6 +971,7 @@ struct heap_page { /* If set, the object is not movable */ bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT]; + bits_t age_bits[HEAP_PAGE_BITMAP_LIMIT * RVALUE_AGE_BIT_COUNT]; }; /* @@ -1011,6 +1015,35 @@ asan_unlock_freelist(struct heap_page *page) #define GC_SWEEP_PAGES_FREEABLE_PER_STEP 3 +#define RVALUE_AGE_BITMAP_INDEX(n) (NUM_IN_PAGE(n) / (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) +#define RVALUE_AGE_BITMAP_OFFSET(n) ((NUM_IN_PAGE(n) % (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) * RVALUE_AGE_BIT_COUNT) + +#define RVALUE_OLD_AGE 3 + +static int +RVALUE_AGE_GET(VALUE obj) +{ + bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits; + return (int)(age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] >> RVALUE_AGE_BITMAP_OFFSET(obj)) & RVALUE_AGE_BIT_MASK; +} + +static void +RVALUE_AGE_SET(VALUE obj, int age) +{ + RUBY_ASSERT(age <= RVALUE_OLD_AGE); + bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits; + // clear the bits + age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] &= ~(RVALUE_AGE_BIT_MASK << (RVALUE_AGE_BITMAP_OFFSET(obj))); + // shift the correct value in + age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] |= ((bits_t)age << RVALUE_AGE_BITMAP_OFFSET(obj)); + if (age == RVALUE_OLD_AGE) { + RB_FL_SET_RAW(obj, RUBY_FL_PROMOTED); + } + else { + RB_FL_UNSET_RAW(obj, RUBY_FL_PROMOTED); + } +} + /* Aliases */ #define rb_objspace (*rb_objspace_of(GET_VM())) #define rb_objspace_of(vm) ((vm)->objspace) @@ -1485,8 +1518,6 @@ asan_poison_object_restore(VALUE obj, void *ptr) #define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj)) #define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj)) -#define RVALUE_OLD_AGE 3 -#define RVALUE_AGE_SHIFT 5 /* FL_PROMOTED0 bit */ static int rgengc_remembered(rb_objspace_t *objspace, VALUE obj); static int rgengc_remembered_sweep(rb_objspace_t *objspace, VALUE obj); @@ -1494,12 +1525,6 @@ static int rgengc_remember(rb_objspace_t *objspace, VALUE obj); static void rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap); static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap); -static inline int -RVALUE_FLAGS_AGE(VALUE flags) -{ - return (int)((flags & (FL_PROMOTED0 | FL_PROMOTED1)) >> RVALUE_AGE_SHIFT); -} - static int check_rvalue_consistency_force(const VALUE obj, int terminate) { @@ -1539,7 +1564,7 @@ check_rvalue_consistency_force(const VALUE obj, int terminate) const int mark_bit = RVALUE_MARK_BITMAP(obj) != 0; const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0; const int remembered_bit = MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0; - const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags); + const int age = RVALUE_AGE_GET((VALUE)obj); if (GET_HEAP_PAGE(obj)->flags.in_tomb) { fprintf(stderr, "check_rvalue_consistency: %s is in tomb page.\n", obj_info(obj)); @@ -1683,28 +1708,15 @@ RVALUE_UNCOLLECTIBLE(VALUE obj) } static inline int -RVALUE_OLD_P_RAW(VALUE obj) -{ - const VALUE promoted = FL_PROMOTED0 | FL_PROMOTED1; - return (RBASIC(obj)->flags & promoted) == promoted; -} - -static inline int RVALUE_OLD_P(VALUE obj) { + GC_ASSERT(!RB_SPECIAL_CONST_P(obj)); check_rvalue_consistency(obj); - return RVALUE_OLD_P_RAW(obj); + // Because this will only ever be called on GC controlled objects, + // we can use the faster _RAW function here + return RB_OBJ_PROMOTED_RAW(obj); } -#if RGENGC_CHECK_MODE || GC_DEBUG -static inline int -RVALUE_AGE(VALUE obj) -{ - check_rvalue_consistency(obj); - return RVALUE_FLAGS_AGE(RBASIC(obj)->flags); -} -#endif - static inline void RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *page, VALUE obj) { @@ -1725,39 +1737,39 @@ RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, VALUE obj) RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, GET_HEAP_PAGE(obj), obj); } -static inline VALUE -RVALUE_FLAGS_AGE_SET(VALUE flags, int age) -{ - flags &= ~(FL_PROMOTED0 | FL_PROMOTED1); - flags |= (age << RVALUE_AGE_SHIFT); - return flags; -} - /* set age to age+1 */ static inline void RVALUE_AGE_INC(rb_objspace_t *objspace, VALUE obj) { - VALUE flags = RBASIC(obj)->flags; - int age = RVALUE_FLAGS_AGE(flags); + int age = RVALUE_AGE_GET((VALUE)obj); if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) { rb_bug("RVALUE_AGE_INC: can not increment age of OLD object %s.", obj_info(obj)); } age++; - RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(flags, age); + RVALUE_AGE_SET(obj, age); if (age == RVALUE_OLD_AGE) { RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj); } + check_rvalue_consistency(obj); } static inline void -RVALUE_DEMOTE_RAW(rb_objspace_t *objspace, VALUE obj) +RVALUE_AGE_SET_CANDIDATE(rb_objspace_t *objspace, VALUE obj) { - RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0); - CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj); + check_rvalue_consistency(obj); + GC_ASSERT(!RVALUE_OLD_P(obj)); + RVALUE_AGE_SET(obj, RVALUE_OLD_AGE - 1); + check_rvalue_consistency(obj); +} + +static inline void +RVALUE_AGE_RESET(VALUE obj) +{ + RVALUE_AGE_SET(obj, 0); } static inline void @@ -1770,7 +1782,8 @@ RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj) CLEAR_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj); } - RVALUE_DEMOTE_RAW(objspace, obj); + CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj); + RVALUE_AGE_RESET(obj); if (RVALUE_MARKED(obj)) { objspace->rgengc.old_objects--; @@ -1779,22 +1792,6 @@ RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj) check_rvalue_consistency(obj); } -static inline void -RVALUE_AGE_RESET_RAW(VALUE obj) -{ - RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0); -} - -static inline void -RVALUE_AGE_RESET(VALUE obj) -{ - check_rvalue_consistency(obj); - GC_ASSERT(!RVALUE_OLD_P(obj)); - - RVALUE_AGE_RESET_RAW(obj); - check_rvalue_consistency(obj); -} - static inline int RVALUE_BLACK_P(VALUE obj) { @@ -1962,6 +1959,8 @@ heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj page->freelist = p; asan_lock_freelist(page); + RVALUE_AGE_RESET(obj); + if (RGENGC_CHECK_MODE && /* obj should belong to page */ !(page->start <= (uintptr_t)obj && @@ -2505,6 +2504,11 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, p->as.basic.flags = flags; *((VALUE *)&p->as.basic.klass) = klass; + int t = flags & RUBY_T_MASK; + if (t == T_CLASS || t == T_MODULE || t == T_ICLASS) { + RVALUE_AGE_SET_CANDIDATE(objspace, obj); + } + #if RACTOR_CHECK_MODE rb_ractor_setup_belonging(obj); #endif @@ -2521,12 +2525,6 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, GC_ASSERT(RVALUE_OLD_P(obj) == FALSE); GC_ASSERT(RVALUE_WB_UNPROTECTED(obj) == FALSE); - if (flags & FL_PROMOTED1) { - if (RVALUE_AGE(obj) != 2) rb_bug("newobj: %s of age (%d) != 2.", obj_info(obj), RVALUE_AGE(obj)); - } - else { - if (RVALUE_AGE(obj) > 0) rb_bug("newobj: %s of age (%d) > 0.", obj_info(obj), RVALUE_AGE(obj)); - } if (rgengc_remembered(objspace, (VALUE)obj)) rb_bug("newobj: %s is remembered.", obj_info(obj)); } RB_VM_LOCK_LEAVE_NO_BARRIER(); @@ -9017,7 +9015,7 @@ rb_copy_wb_protected_attribute(VALUE dest, VALUE obj) if (RVALUE_WB_UNPROTECTED(obj) && !RVALUE_WB_UNPROTECTED(dest)) { if (!RVALUE_OLD_P(dest)) { MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest); - RVALUE_AGE_RESET_RAW(dest); + RVALUE_AGE_RESET(dest); } else { RVALUE_DEMOTE(objspace, dest); @@ -9789,6 +9787,7 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s int marked; int wb_unprotected; int uncollectible; + int age; RVALUE *dest = (RVALUE *)free; RVALUE *src = (RVALUE *)scan; @@ -9804,6 +9803,7 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s wb_unprotected = RVALUE_WB_UNPROTECTED((VALUE)src); uncollectible = RVALUE_UNCOLLECTIBLE((VALUE)src); bool remembered = RVALUE_REMEMBERED((VALUE)src); + age = RVALUE_AGE_GET((VALUE)src); /* Clear bits for eventual T_MOVED */ CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS((VALUE)src), (VALUE)src); @@ -9846,6 +9846,7 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s } memset(src, 0, src_slot_size); + RVALUE_AGE_RESET((VALUE)src); /* Set bits for object in new location */ if (remembered) { @@ -9876,6 +9877,7 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS((VALUE)dest), (VALUE)dest); } + RVALUE_AGE_SET((VALUE)dest, age); /* Assign forwarding address */ src->as.moved.flags = T_MOVED; src->as.moved.dummy = Qundef; @@ -13439,7 +13441,7 @@ rb_raw_obj_info_common(char *const buff, const size_t buff_size, const VALUE obj } } else { - const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags); + const int age = RVALUE_AGE_GET(obj); if (is_pointer_to_heap(&rb_objspace, (void *)obj)) { APPEND_F("%p [%d%s%s%s%s%s%s] %s ", @@ -13776,7 +13778,7 @@ rb_gcdebug_print_obj_condition(VALUE obj) fprintf(stderr, "marked? : %s\n", MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj) ? "true" : "false"); fprintf(stderr, "pinned? : %s\n", MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj) ? "true" : "false"); - fprintf(stderr, "age? : %d\n", RVALUE_AGE(obj)); + fprintf(stderr, "age? : %d\n", RVALUE_AGE_GET(obj)); fprintf(stderr, "old? : %s\n", RVALUE_OLD_P(obj) ? "true" : "false"); fprintf(stderr, "WB-protected?: %s\n", RVALUE_WB_UNPROTECTED(obj) ? "false" : "true"); fprintf(stderr, "remembered? : %s\n", RVALUE_REMEMBERED(obj) ? "true" : "false"); diff --git a/include/ruby/internal/core/rtypeddata.h b/include/ruby/internal/core/rtypeddata.h index 1abf965305..c7904746fd 100644 --- a/include/ruby/internal/core/rtypeddata.h +++ b/include/ruby/internal/core/rtypeddata.h @@ -173,10 +173,9 @@ rbimpl_typeddata_flags { RUBY_TYPED_WB_PROTECTED = RUBY_FL_WB_PROTECTED, /* THIS FLAG DEPENDS ON Ruby version */ /** - * This flag is mysterious. It seems nobody is currently using it. The - * intention of this flag is also unclear. We need further investigations. + * This flag no longer in use */ - RUBY_TYPED_PROMOTED1 = RUBY_FL_PROMOTED1, /* THIS FLAG DEPENDS ON Ruby version */ + RUBY_TYPED_UNUSED = RUBY_FL_UNUSED6, /** * This flag determines whether marking and compaction should be carried out diff --git a/include/ruby/internal/fl_type.h b/include/ruby/internal/fl_type.h index d59c258a3f..f7f5abdd9b 100644 --- a/include/ruby/internal/fl_type.h +++ b/include/ruby/internal/fl_type.h @@ -57,8 +57,7 @@ #define FL_SINGLETON RBIMPL_CAST((VALUE)RUBY_FL_SINGLETON) /**< @old{RUBY_FL_SINGLETON} */ #define FL_WB_PROTECTED RBIMPL_CAST((VALUE)RUBY_FL_WB_PROTECTED) /**< @old{RUBY_FL_WB_PROTECTED} */ -#define FL_PROMOTED0 RBIMPL_CAST((VALUE)RUBY_FL_PROMOTED0) /**< @old{RUBY_FL_PROMOTED0} */ -#define FL_PROMOTED1 RBIMPL_CAST((VALUE)RUBY_FL_PROMOTED1) /**< @old{RUBY_FL_PROMOTED1} */ +#define FL_PROMOTED RBIMPL_CAST((VALUE)RUBY_FL_PROMOTED) /**< @old{RUBY_FL_PROMOTED} */ #define FL_FINALIZE RBIMPL_CAST((VALUE)RUBY_FL_FINALIZE) /**< @old{RUBY_FL_FINALIZE} */ #define FL_TAINT RBIMPL_CAST((VALUE)RUBY_FL_TAINT) /**< @old{RUBY_FL_TAINT} */ #define FL_SHAREABLE RBIMPL_CAST((VALUE)RUBY_FL_SHAREABLE) /**< @old{RUBY_FL_SHAREABLE} */ @@ -200,12 +199,15 @@ ruby_fl_type { RUBY_FL_WB_PROTECTED = (1<<5), /** - * This flag has something to do with our garbage collector. These days - * ruby objects are "generational". There are those who are young and - * those who are old. Young objects are prone to die; monitored relatively - * extensively by the garbage collector. OTOH old objects tend to live - * longer. They are relatively rarely considered. This flag is set when a - * object experienced promotion i.e. survived a garbage collection. + * Ruby objects are "generational". There are young objects & old objects. + * Young objects are prone to die & monitored relatively extensively by the + * garbage collector. Old objects tend to live longer & are monitored less + * frequently. When an object survives a GC, its age is incremented. When + * age is equal to RVALUE_OLD_AGE, the object becomes Old. This flag is set + * when an object becomes old, and is used by the write barrier to check if + * an old object should be considered for marking more frequently - as old + * objects that have references added between major GCs need to be remarked + * to prevent the referred object being mistakenly swept. * * @internal * @@ -213,41 +215,14 @@ ruby_fl_type { * 3rd parties. It must be an implementation detail that they should never * know. Might better be hidden. */ - RUBY_FL_PROMOTED0 = (1<<5), + RUBY_FL_PROMOTED = (1<<5), /** - * This flag has something to do with our garbage collector. These days - * ruby objects are "generational". There are those who are young and - * those who are old. Young objects are prone to die; monitored relatively - * extensively by the garbage collector. OTOH old objects tend to live - * longer. They are relatively rarely considered. This flag is set when a - * object experienced two promotions i.e. survived garbage collections - * twice. + * This flag is no longer in use * * @internal - * - * But honestly, @shyouhei doesn't think this flag should be visible from - * 3rd parties. It must be an implementation detail that they should never - * know. Might better be hidden. - */ - RUBY_FL_PROMOTED1 = (1<<6), - - /** - * This flag has something to do with our garbage collector. These days - * ruby objects are "generational". There are those who are young and - * those who are old. Young objects are prone to die; monitored relatively - * extensively by the garbage collector. OTOH old objects tend to live - * longer. They are relatively rarely considered. This flag is set when a - * object experienced promotions i.e. survived more than one garbage - * collections. - * - * @internal - * - * But honestly, @shyouhei doesn't think this flag should be visible from - * 3rd parties. It must be an implementation detail that they should never - * know. Might better be hidden. */ - RUBY_FL_PROMOTED = RUBY_FL_PROMOTED0 | RUBY_FL_PROMOTED1, + RUBY_FL_UNUSED6 = (1<<6), /** * This flag has something to do with finalisers. A ruby object can have @@ -70,7 +70,7 @@ CLONESETUP(VALUE clone, VALUE obj) RBIMPL_ASSERT_OR_ASSUME(! RB_SPECIAL_CONST_P(obj)); RBIMPL_ASSERT_OR_ASSUME(! RB_SPECIAL_CONST_P(clone)); - const VALUE flags = RUBY_FL_PROMOTED0 | RUBY_FL_PROMOTED1 | RUBY_FL_FINALIZE; + const VALUE flags = RUBY_FL_PROMOTED | RUBY_FL_FINALIZE; rb_obj_setup(clone, rb_singleton_class_clone(obj), RB_FL_TEST_RAW(obj, ~flags)); rb_singleton_class_attached(RBASIC_CLASS(clone), clone); diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index 553f9cdbea..bbf1578a45 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -192,9 +192,8 @@ pub type ruby_value_type = u32; pub const RUBY_FL_USHIFT: ruby_fl_ushift = 12; pub type ruby_fl_ushift = u32; pub const RUBY_FL_WB_PROTECTED: ruby_fl_type = 32; -pub const RUBY_FL_PROMOTED0: ruby_fl_type = 32; -pub const RUBY_FL_PROMOTED1: ruby_fl_type = 64; -pub const RUBY_FL_PROMOTED: ruby_fl_type = 96; +pub const RUBY_FL_PROMOTED: ruby_fl_type = 32; +pub const RUBY_FL_UNUSED6: ruby_fl_type = 64; pub const RUBY_FL_FINALIZE: ruby_fl_type = 128; pub const RUBY_FL_TAINT: ruby_fl_type = 0; pub const RUBY_FL_SHAREABLE: ruby_fl_type = 256; |