1 files changed, 102 insertions, 171 deletions
diff --git a/gc.c b/gc.c
index 3e7d88209f..b0876fca5e 100644
--- a/gc.c
+++ b/gc.c
@@ -131,45 +131,45 @@
 #include "shape.h"
 
 unsigned int
-rb_gc_vm_lock(void)
+rb_gc_vm_lock(const char *file, int line)
 {
     unsigned int lev = 0;
-    RB_VM_LOCK_ENTER_LEV(&lev);
+    rb_vm_lock_enter(&lev, file, line);
     return lev;
 }
 
 void
-rb_gc_vm_unlock(unsigned int lev)
+rb_gc_vm_unlock(unsigned int lev, const char *file, int line)
 {
-    RB_VM_LOCK_LEAVE_LEV(&lev);
+    rb_vm_lock_leave(&lev, file, line);
 }
 
 unsigned int
-rb_gc_cr_lock(void)
+rb_gc_cr_lock(const char *file, int line)
 {
     unsigned int lev;
-    RB_VM_LOCK_ENTER_CR_LEV(GET_RACTOR(), &lev);
+    rb_vm_lock_enter_cr(GET_RACTOR(), &lev, file, line);
     return lev;
 }
 
 void
-rb_gc_cr_unlock(unsigned int lev)
+rb_gc_cr_unlock(unsigned int lev, const char *file, int line)
 {
-    RB_VM_LOCK_LEAVE_CR_LEV(GET_RACTOR(), &lev);
+    rb_vm_lock_leave_cr(GET_RACTOR(), &lev, file, line);
 }
 
 unsigned int
-rb_gc_vm_lock_no_barrier(void)
+rb_gc_vm_lock_no_barrier(const char *file, int line)
 {
     unsigned int lev = 0;
-    RB_VM_LOCK_ENTER_LEV_NB(&lev);
+    rb_vm_lock_enter_nb(&lev, file, line);
     return lev;
 }
 
 void
-rb_gc_vm_unlock_no_barrier(unsigned int lev)
+rb_gc_vm_unlock_no_barrier(unsigned int lev, const char *file, int line)
 {
-    RB_VM_LOCK_LEAVE_LEV_NB(&lev);
+    rb_vm_lock_leave_nb(&lev, file, line);
 }
 
 void
@@ -1201,7 +1201,6 @@ rb_data_free(void *objspace, VALUE obj)
 
 struct classext_foreach_args {
     VALUE klass;
-    bool obj_too_complex;
     rb_objspace_t *objspace; // used for update_*
 };
 
@@ -1213,12 +1212,6 @@ classext_free(rb_classext_t *ext, bool is_prime, VALUE namespace, void *arg)
 
     rb_id_table_free(RCLASSEXT_M_TBL(ext));
     rb_cc_tbl_free(RCLASSEXT_CC_TBL(ext), args->klass);
-    if (args->obj_too_complex) {
-        st_free_table((st_table *)RCLASSEXT_FIELDS(ext));
-    }
-    else {
-        xfree(RCLASSEXT_FIELDS(ext));
-    }
     if (!RCLASSEXT_SHARED_CONST_TBL(ext) && (tbl = RCLASSEXT_CONST_TBL(ext)) != NULL) {
         rb_free_const_table(tbl);
     }
@@ -1292,8 +1285,6 @@ rb_gc_obj_free(void *objspace, VALUE obj)
       case T_MODULE:
       case T_CLASS:
         args.klass = obj;
-        args.obj_too_complex = rb_shape_obj_too_complex_p(obj) ? true : false;
-
         rb_class_classext_foreach(obj, classext_free, (void *)&args);
         if (RCLASS(obj)->ns_classext_tbl) {
             st_free_table(RCLASS(obj)->ns_classext_tbl);
@@ -1783,9 +1774,9 @@ generate_next_object_id(void)
     // 64bit atomics are available
     return SIZET2NUM(RUBY_ATOMIC_SIZE_FETCH_ADD(object_id_counter, 1) * OBJ_ID_INCREMENT);
 #else
-    unsigned int lock_lev = rb_gc_vm_lock();
+    unsigned int lock_lev = RB_GC_VM_LOCK();
     VALUE id = ULL2NUM(++object_id_counter * OBJ_ID_INCREMENT);
-    rb_gc_vm_unlock(lock_lev);
+    RB_GC_VM_UNLOCK(lock_lev);
     return id;
 #endif
 }
@@ -1867,7 +1858,7 @@ class_object_id(VALUE klass)
 {
     VALUE id = RUBY_ATOMIC_VALUE_LOAD(RCLASS(klass)->object_id);
     if (!id) {
-        unsigned int lock_lev = rb_gc_vm_lock();
+        unsigned int lock_lev = RB_GC_VM_LOCK();
         id = generate_next_object_id();
         VALUE existing_id = RUBY_ATOMIC_VALUE_CAS(RCLASS(klass)->object_id, 0, id);
         if (existing_id) {
@@ -1876,7 +1867,7 @@ class_object_id(VALUE klass)
         else if (RB_UNLIKELY(id2ref_tbl)) {
             st_insert(id2ref_tbl, id, klass);
         }
-        rb_gc_vm_unlock(lock_lev);
+        RB_GC_VM_UNLOCK(lock_lev);
     }
     return id;
 }
@@ -1946,9 +1937,9 @@ object_id(VALUE obj)
     }
 
     if (UNLIKELY(rb_gc_multi_ractor_p() && rb_ractor_shareable_p(obj))) {
-        unsigned int lock_lev = rb_gc_vm_lock();
+        unsigned int lock_lev = RB_GC_VM_LOCK();
         VALUE id = object_id0(obj);
-        rb_gc_vm_unlock(lock_lev);
+        RB_GC_VM_UNLOCK(lock_lev);
         return id;
     }
 
@@ -1983,7 +1974,7 @@ object_id_to_ref(void *objspace_ptr, VALUE object_id)
 {
     rb_objspace_t *objspace = objspace_ptr;
 
-    unsigned int lev = rb_gc_vm_lock();
+    unsigned int lev = RB_GC_VM_LOCK();
 
     if (!id2ref_tbl) {
         rb_gc_vm_barrier(); // stop other ractors
@@ -2007,7 +1998,7 @@ object_id_to_ref(void *objspace_ptr, VALUE object_id)
     VALUE obj;
     bool found = st_lookup(id2ref_tbl, object_id, &obj) && !rb_gc_impl_garbage_object_p(objspace, obj);
 
-    rb_gc_vm_unlock(lev);
+    RB_GC_VM_UNLOCK(lev);
 
     if (found) {
         return obj;
@@ -2024,27 +2015,6 @@ object_id_to_ref(void *objspace_ptr, VALUE object_id)
 static inline void
 obj_free_object_id(VALUE obj)
 {
-    if (RB_BUILTIN_TYPE(obj) == T_IMEMO) {
-        return;
-    }
-
-#if RUBY_DEBUG
-    switch (BUILTIN_TYPE(obj)) {
-      case T_CLASS:
-      case T_MODULE:
-        break;
-      default:
-        if (rb_shape_obj_has_id(obj)) {
-            VALUE id = object_id_get(obj, RBASIC_SHAPE_ID(obj)); // Crash if missing
-            if (!(FIXNUM_P(id) || RB_TYPE_P(id, T_BIGNUM))) {
-                rb_p(obj);
-                rb_bug("Corrupted object_id");
-            }
-        }
-        break;
-    }
-#endif
-
     VALUE obj_id = 0;
     if (RB_UNLIKELY(id2ref_tbl)) {
         switch (BUILTIN_TYPE(obj)) {
@@ -2052,21 +2022,32 @@ obj_free_object_id(VALUE obj)
           case T_MODULE:
             obj_id = RCLASS(obj)->object_id;
             break;
-          default: {
+          case T_IMEMO:
+            if (!IMEMO_TYPE_P(obj, imemo_fields)) {
+                return;
+            }
+            // fallthrough
+          case T_OBJECT:
+            {
             shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
             if (rb_shape_has_object_id(shape_id)) {
                 obj_id = object_id_get(obj, shape_id);
             }
             break;
           }
+          default:
+            // For generic_fields, the T_IMEMO/fields is responsible for freeing the id.
+            return;
         }
 
         if (RB_UNLIKELY(obj_id)) {
             RUBY_ASSERT(FIXNUM_P(obj_id) || RB_TYPE_P(obj_id, T_BIGNUM));
 
             if (!st_delete(id2ref_tbl, (st_data_t *)&obj_id, NULL)) {
-                // If we're currently building the table then it's not a bug
-                if (id2ref_tbl_built) {
+                // If we're currently building the table then it's not a bug.
+                // The the object is a T_IMEMO/fields, then it's possible the actual object
+                // has been garbage collected already.
+                if (id2ref_tbl_built && !RB_TYPE_P(obj, T_IMEMO)) {
                     rb_bug("Object ID seen, but not in _id2ref table: object_id=%llu object=%s", NUM2ULL(obj_id), rb_obj_info(obj));
                 }
             }
@@ -2079,9 +2060,8 @@ rb_gc_obj_free_vm_weak_references(VALUE obj)
 {
     obj_free_object_id(obj);
 
-    if (FL_TEST_RAW(obj, FL_EXIVAR)) {
-        rb_free_generic_ivar((VALUE)obj);
-        FL_UNSET_RAW(obj, FL_EXIVAR);
+    if (rb_obj_exivar_p(obj)) {
+        rb_free_generic_ivar(obj);
     }
 
     switch (BUILTIN_TYPE(obj)) {
@@ -2095,6 +2075,15 @@ rb_gc_obj_free_vm_weak_references(VALUE obj)
         break;
       case T_IMEMO:
         switch (imemo_type(obj)) {
+          case imemo_callcache: {
+            const struct rb_callcache *cc = (const struct rb_callcache *)obj;
+
+            if (vm_cc_refinement_p(cc)) {
+                rb_vm_delete_cc_refinement(cc);
+            }
+
+            break;
+          }
           case imemo_callinfo:
             rb_vm_ci_free((const struct rb_callinfo *)obj);
             break;
@@ -2297,18 +2286,6 @@ classext_memsize(rb_classext_t *ext, bool prime, VALUE namespace, void *arg)
 }
 
 static void
-classext_fields_hash_memsize(rb_classext_t *ext, bool prime, VALUE namespace, void *arg)
-{
-    size_t *size = (size_t *)arg;
-    size_t count;
-    RB_VM_LOCKING() {
-        count = rb_st_table_size((st_table *)RCLASSEXT_FIELDS(ext));
-    }
-    // class IV sizes are allocated as powers of two
-    *size += SIZEOF_VALUE << bit_length(count);
-}
-
-static void
 classext_superclasses_memsize(rb_classext_t *ext, bool prime, VALUE namespace, void *arg)
 {
     size_t *size = (size_t *)arg;
@@ -2329,10 +2306,6 @@ rb_obj_memsize_of(VALUE obj)
         return 0;
     }
 
-    if (FL_TEST(obj, FL_EXIVAR)) {
-        size += rb_generic_ivar_memsize(obj);
-    }
-
     switch (BUILTIN_TYPE(obj)) {
       case T_OBJECT:
         if (rb_shape_obj_too_complex_p(obj)) {
@@ -2345,15 +2318,6 @@ rb_obj_memsize_of(VALUE obj)
       case T_MODULE:
       case T_CLASS:
         rb_class_classext_foreach(obj, classext_memsize, (void *)&size);
-
-        if (rb_shape_obj_too_complex_p(obj)) {
-            rb_class_classext_foreach(obj, classext_fields_hash_memsize, (void *)&size);
-        }
-        else {
-            // class IV sizes are allocated as powers of two
-            size += SIZEOF_VALUE << bit_length(RCLASS_FIELDS_COUNT(obj));
-        }
-
         rb_class_classext_foreach(obj, classext_superclasses_memsize, (void *)&size);
         break;
       case T_ICLASS:
@@ -3082,7 +3046,6 @@ rb_gc_mark_roots(void *objspace, const char **categoryp)
 
     MARK_CHECKPOINT("vm");
     rb_vm_mark(vm);
-    if (vm->self) gc_mark_internal(vm->self);
 
     MARK_CHECKPOINT("end_proc");
     rb_mark_end_proc();
@@ -3126,10 +3089,7 @@ gc_mark_classext_module(rb_classext_t *ext, bool prime, VALUE namespace, void *a
         gc_mark_internal(RCLASSEXT_SUPER(ext));
     }
     mark_m_tbl(objspace, RCLASSEXT_M_TBL(ext));
-    if (rb_shape_obj_too_complex_p(obj)) {
-        gc_mark_tbl_no_pin((st_table *)RCLASSEXT_FIELDS(ext));
-        // for the case ELSE is written in rb_gc_mark_children() because it's per RClass, not classext
-    }
+    gc_mark_internal(RCLASSEXT_FIELDS_OBJ(ext));
     if (!RCLASSEXT_SHARED_CONST_TBL(ext) && RCLASSEXT_CONST_TBL(ext)) {
         mark_const_tbl(objspace, RCLASSEXT_CONST_TBL(ext));
     }
@@ -3166,7 +3126,7 @@ rb_gc_mark_children(void *objspace, VALUE obj)
 {
     struct gc_mark_classext_foreach_arg foreach_args;
 
-    if (FL_TEST_RAW(obj, FL_EXIVAR)) {
+    if (rb_obj_exivar_p(obj)) {
         rb_mark_generic_ivar(obj);
     }
 
@@ -3209,12 +3169,6 @@ rb_gc_mark_children(void *objspace, VALUE obj)
         foreach_args.objspace = objspace;
         foreach_args.obj = obj;
         rb_class_classext_foreach(obj, gc_mark_classext_module, (void *)&foreach_args);
-
-        if (!rb_shape_obj_too_complex_p(obj)) {
-            for (attr_index_t i = 0; i < RCLASS_FIELDS_COUNT(obj); i++) {
-                gc_mark_internal(RCLASS_PRIME_FIELDS(obj)[i]);
-            }
-        }
         break;
 
       case T_ICLASS:
@@ -3840,7 +3794,6 @@ static void
 update_classext(rb_classext_t *ext, bool is_prime, VALUE namespace, void *arg)
 {
     struct classext_foreach_args *args = (struct classext_foreach_args *)arg;
-    VALUE klass = args->klass;
     rb_objspace_t *objspace = args->objspace;
 
     if (RCLASSEXT_SUPER(ext)) {
@@ -3849,16 +3802,7 @@ update_classext(rb_classext_t *ext, bool is_prime, VALUE namespace, void *arg)
 
     update_m_tbl(objspace, RCLASSEXT_M_TBL(ext));
 
-    if (args->obj_too_complex) {
-        gc_ref_update_table_values_only((st_table *)RCLASSEXT_FIELDS(ext));
-    }
-    else {
-        // Classext is not copied in this case
-        for (attr_index_t i = 0; i < RCLASS_FIELDS_COUNT(klass); i++) {
-            UPDATE_IF_MOVED(objspace, RCLASSEXT_FIELDS(RCLASS_EXT_PRIME(klass))[i]);
-        }
-    }
-
+    UPDATE_IF_MOVED(objspace, ext->fields_obj);
     if (!RCLASSEXT_SHARED_CONST_TBL(ext)) {
         update_const_tbl(objspace, RCLASSEXT_CONST_TBL(ext));
     }
@@ -3930,6 +3874,23 @@ vm_weak_table_foreach_update_weak_key(st_data_t *key, st_data_t *value, st_data_
 }
 
 static int
+vm_weak_table_cc_refinement_foreach(st_data_t key, st_data_t data, int error)
+{
+    struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
+
+    return iter_data->callback((VALUE)key, iter_data->data);
+}
+
+static int
+vm_weak_table_cc_refinement_foreach_update_update(st_data_t *key, st_data_t data, int existing)
+{
+    struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
+
+    return iter_data->update_callback((VALUE *)key, iter_data->data);
+}
+
+
+static int
 vm_weak_table_str_sym_foreach(st_data_t key, st_data_t value, st_data_t data, int error)
 {
     struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
@@ -3960,38 +3921,6 @@ vm_weak_table_foreach_update_weak_value(st_data_t *key, st_data_t *value, st_dat
     return iter_data->update_callback((VALUE *)value, iter_data->data);
 }
 
-static void
-free_gen_fields_tbl(VALUE obj, struct gen_fields_tbl *fields_tbl)
-{
-    if (UNLIKELY(rb_shape_obj_too_complex_p(obj))) {
-        st_free_table(fields_tbl->as.complex.table);
-    }
-
-    xfree(fields_tbl);
-}
-
-static int
-vm_weak_table_gen_fields_foreach_too_complex_i(st_data_t _key, st_data_t value, st_data_t data, int error)
-{
-    struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
-
-    GC_ASSERT(!iter_data->weak_only);
-
-    if (SPECIAL_CONST_P((VALUE)value)) return ST_CONTINUE;
-
-    return iter_data->callback((VALUE)value, iter_data->data);
-}
-
-static int
-vm_weak_table_gen_fields_foreach_too_complex_replace_i(st_data_t *_key, st_data_t *value, st_data_t data, int existing)
-{
-    struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
-
-    GC_ASSERT(!iter_data->weak_only);
-
-    return iter_data->update_callback((VALUE *)value, iter_data->data);
-}
-
 struct st_table *rb_generic_fields_tbl_get(void);
 
 static int
@@ -4028,60 +3957,50 @@ vm_weak_table_gen_fields_foreach(st_data_t key, st_data_t value, st_data_t data)
 
     int ret = iter_data->callback((VALUE)key, iter_data->data);
 
+    VALUE new_value = (VALUE)value;
+    VALUE new_key = (VALUE)key;
+
     switch (ret) {
       case ST_CONTINUE:
         break;
 
       case ST_DELETE:
-        free_gen_fields_tbl((VALUE)key, (struct gen_fields_tbl *)value);
-
-        FL_UNSET((VALUE)key, FL_EXIVAR);
+        RBASIC_SET_SHAPE_ID((VALUE)key, ROOT_SHAPE_ID);
         return ST_DELETE;
 
       case ST_REPLACE: {
-        VALUE new_key = (VALUE)key;
         ret = iter_data->update_callback(&new_key, iter_data->data);
-        if (key != new_key) ret = ST_DELETE;
-        DURING_GC_COULD_MALLOC_REGION_START();
-        {
-            st_insert(rb_generic_fields_tbl_get(), (st_data_t)new_key, value);
+        if (key != new_key) {
+            ret = ST_DELETE;
         }
-        DURING_GC_COULD_MALLOC_REGION_END();
-        key = (st_data_t)new_key;
         break;
       }
 
       default:
-        return ret;
+        rb_bug("vm_weak_table_gen_fields_foreach: return value %d not supported", ret);
     }
 
     if (!iter_data->weak_only) {
-        struct gen_fields_tbl *fields_tbl = (struct gen_fields_tbl *)value;
+        int ivar_ret = iter_data->callback(new_value, iter_data->data);
+        switch (ivar_ret) {
+          case ST_CONTINUE:
+            break;
 
-        if (rb_shape_obj_too_complex_p((VALUE)key)) {
-            st_foreach_with_replace(
-                fields_tbl->as.complex.table,
-                vm_weak_table_gen_fields_foreach_too_complex_i,
-                vm_weak_table_gen_fields_foreach_too_complex_replace_i,
-                data
-            );
+          case ST_REPLACE:
+            iter_data->update_callback(&new_value, iter_data->data);
+            break;
+
+          default:
+            rb_bug("vm_weak_table_gen_fields_foreach: return value %d not supported", ivar_ret);
         }
-        else {
-            for (uint32_t i = 0; i < fields_tbl->as.shape.fields_count; i++) {
-                if (SPECIAL_CONST_P(fields_tbl->as.shape.fields[i])) continue;
+    }
 
-                int ivar_ret = iter_data->callback(fields_tbl->as.shape.fields[i], iter_data->data);
-                switch (ivar_ret) {
-                  case ST_CONTINUE:
-                    break;
-                  case ST_REPLACE:
-                    iter_data->update_callback(&fields_tbl->as.shape.fields[i], iter_data->data);
-                    break;
-                  default:
-                    rb_bug("vm_weak_table_gen_fields_foreach: return value %d not supported", ivar_ret);
-                }
-            }
+    if (key != new_key || value != new_value) {
+        DURING_GC_COULD_MALLOC_REGION_START();
+        {
+            st_insert(rb_generic_fields_tbl_get(), (st_data_t)new_key, new_value);
         }
+        DURING_GC_COULD_MALLOC_REGION_END();
     }
 
     return ret;
@@ -4178,8 +4097,21 @@ rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func callback,
         );
         break;
       }
+      case RB_GC_VM_CC_REFINEMENT_TABLE: {
+        if (vm->cc_refinement_table) {
+            set_foreach_with_replace(
+              vm->cc_refinement_table,
+              vm_weak_table_cc_refinement_foreach,
+              vm_weak_table_cc_refinement_foreach_update_update,
+              (st_data_t)&foreach_data
+            );
+        }
+        break;
+      }
       case RB_GC_VM_WEAK_TABLE_COUNT:
-        rb_bug("Unreacheable");
+        rb_bug("Unreachable");
+      default:
+        rb_bug("rb_gc_vm_weak_table_foreach: unknown table %d", table);
     }
 }
 
@@ -4215,7 +4147,6 @@ rb_gc_update_object_references(void *objspace, VALUE obj)
         // Continue to the shared T_CLASS/T_MODULE
       case T_MODULE:
         args.klass = obj;
-        args.obj_too_complex = rb_shape_obj_too_complex_p(obj);
         args.objspace = objspace;
         rb_class_classext_foreach(obj, update_classext, (void *)&args);
         break;