summaryrefslogtreecommitdiff
path: root/variable.c
diff options
context:
space:
mode:
Diffstat (limited to 'variable.c')
-rw-r--r--variable.c609
1 files changed, 304 insertions, 305 deletions
diff --git a/variable.c b/variable.c
index eb232e52cf..e535aefe27 100644
--- a/variable.c
+++ b/variable.c
@@ -1197,8 +1197,31 @@ rb_generic_fields_tbl_get(void)
return generic_fields_tbl_;
}
+static inline VALUE
+generic_fields_lookup(VALUE obj, ID id, bool force_check_ractor)
+{
+ VALUE fields_obj = Qfalse;
+ RB_VM_LOCKING() {
+ st_table *generic_tbl = generic_fields_tbl(obj, id, false);
+ st_lookup(generic_tbl, obj, (st_data_t *)&fields_obj);
+ }
+ return fields_obj;
+}
+
+static inline void
+generic_fields_insert(VALUE obj, VALUE fields_obj)
+{
+ RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields));
+
+ RB_VM_LOCKING() {
+ st_table *generic_tbl = generic_fields_tbl_no_ractor_check(obj);
+ st_insert(generic_tbl, obj, fields_obj);
+ }
+ RB_OBJ_WRITTEN(obj, Qundef, fields_obj);
+}
+
int
-rb_gen_fields_tbl_get(VALUE obj, ID id, struct gen_fields_tbl **fields_tbl)
+rb_gen_fields_tbl_get(VALUE obj, ID id, VALUE *fields_obj)
{
RUBY_ASSERT(!RB_TYPE_P(obj, T_ICLASS));
@@ -1207,7 +1230,7 @@ rb_gen_fields_tbl_get(VALUE obj, ID id, struct gen_fields_tbl **fields_tbl)
RB_VM_LOCKING() {
if (st_lookup(generic_fields_tbl(obj, id, false), (st_data_t)obj, &data)) {
- *fields_tbl = (struct gen_fields_tbl *)data;
+ *fields_obj = (VALUE)data;
r = 1;
}
}
@@ -1216,33 +1239,17 @@ rb_gen_fields_tbl_get(VALUE obj, ID id, struct gen_fields_tbl **fields_tbl)
}
int
-rb_ivar_generic_fields_tbl_lookup(VALUE obj, struct gen_fields_tbl **fields_tbl)
-{
- return rb_gen_fields_tbl_get(obj, 0, fields_tbl);
-}
-
-static size_t
-gen_fields_tbl_bytes(size_t n)
+rb_ivar_generic_fields_tbl_lookup(VALUE obj, VALUE *fields_obj)
{
- return offsetof(struct gen_fields_tbl, as.shape.fields) + n * sizeof(VALUE);
+ return rb_gen_fields_tbl_get(obj, 0, fields_obj);
}
-
void
rb_mark_generic_ivar(VALUE obj)
{
- st_data_t data;
- if (st_lookup(generic_fields_tbl_no_ractor_check(obj), (st_data_t)obj, &data)) {
- struct gen_fields_tbl *fields_tbl = (struct gen_fields_tbl *)data;
- if (rb_shape_obj_too_complex_p(obj)) {
- rb_mark_tbl_no_pin(fields_tbl->as.complex.table);
- }
- else {
- uint32_t fields_count = RSHAPE_LEN(RBASIC_SHAPE_ID(obj));
- for (uint32_t i = 0; i < fields_count; i++) {
- rb_gc_mark_movable(fields_tbl->as.shape.fields[i]);
- }
- }
+ VALUE data;
+ if (st_lookup(generic_fields_tbl_no_ractor_check(obj), (st_data_t)obj, (st_data_t *)&data)) {
+ rb_gc_mark_movable(data);
}
}
@@ -1252,48 +1259,10 @@ rb_free_generic_ivar(VALUE obj)
if (rb_obj_exivar_p(obj)) {
st_data_t key = (st_data_t)obj, value;
- bool too_complex = rb_shape_obj_too_complex_p(obj);
-
RB_VM_LOCKING() {
- if (st_delete(generic_fields_tbl_no_ractor_check(obj), &key, &value)) {
- struct gen_fields_tbl *fields_tbl = (struct gen_fields_tbl *)value;
-
- if (UNLIKELY(too_complex)) {
- st_free_table(fields_tbl->as.complex.table);
- }
-
- xfree(fields_tbl);
- }
- }
- RBASIC_SET_SHAPE_ID(obj, ROOT_SHAPE_ID);
- }
-}
-
-size_t
-rb_generic_ivar_memsize(VALUE obj)
-{
- struct gen_fields_tbl *fields_tbl;
-
- if (rb_gen_fields_tbl_get(obj, 0, &fields_tbl)) {
- if (rb_shape_obj_too_complex_p(obj)) {
- return sizeof(struct gen_fields_tbl) + st_memsize(fields_tbl->as.complex.table);
- }
- else {
- return gen_fields_tbl_bytes(RSHAPE_CAPACITY(RBASIC_SHAPE_ID(obj)));
+ st_delete(generic_fields_tbl_no_ractor_check(obj), &key, &value);
}
}
- return 0;
-}
-
-static size_t
-gen_fields_tbl_count(VALUE obj, const struct gen_fields_tbl *fields_tbl)
-{
- if (rb_shape_obj_too_complex_p(obj)) {
- return st_table_size(fields_tbl->as.complex.table);
- }
- else {
- return RSHAPE_LEN(RBASIC_SHAPE_ID(obj));
- }
}
VALUE
@@ -1321,12 +1290,16 @@ rb_obj_field_get(VALUE obj, shape_id_t target_shape_id)
case T_OBJECT:
fields_hash = ROBJECT_FIELDS_HASH(obj);
break;
+ case T_IMEMO:
+ RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields));
+ fields_hash = rb_imemo_fields_complex_tbl(obj);
+ break;
default:
RUBY_ASSERT(rb_obj_exivar_p(obj));
- struct gen_fields_tbl *fields_tbl = NULL;
- rb_ivar_generic_fields_tbl_lookup(obj, &fields_tbl);
- RUBY_ASSERT(fields_tbl);
- fields_hash = fields_tbl->as.complex.table;
+ VALUE fields_obj = 0;
+ rb_ivar_generic_fields_tbl_lookup(obj, &fields_obj);
+ RUBY_ASSERT(fields_obj);
+ fields_hash = rb_imemo_fields_complex_tbl(fields_obj);
break;
}
VALUE value = Qundef;
@@ -1352,12 +1325,16 @@ rb_obj_field_get(VALUE obj, shape_id_t target_shape_id)
case T_OBJECT:
fields = ROBJECT_FIELDS(obj);
break;
+ case T_IMEMO:
+ RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields));
+ fields = rb_imemo_fields_ptr(obj);
+ break;
default:
RUBY_ASSERT(rb_obj_exivar_p(obj));
- struct gen_fields_tbl *fields_tbl = NULL;
- rb_ivar_generic_fields_tbl_lookup(obj, &fields_tbl);
- RUBY_ASSERT(fields_tbl);
- fields = fields_tbl->as.shape.fields;
+ VALUE fields_obj = 0;
+ rb_ivar_generic_fields_tbl_lookup(obj, &fields_obj);
+ RUBY_ASSERT(fields_obj);
+ fields = rb_imemo_fields_ptr(fields_obj);
break;
}
return fields[attr_index];
@@ -1393,11 +1370,11 @@ rb_ivar_lookup(VALUE obj, ID id, VALUE undef)
case T_IMEMO:
// Handled like T_OBJECT
{
- RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_class_fields));
+ RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields));
shape_id = RBASIC_SHAPE_ID(obj);
if (rb_shape_too_complex_p(shape_id)) {
- st_table *iv_table = rb_imemo_class_fields_complex_tbl(obj);
+ st_table *iv_table = rb_imemo_fields_complex_tbl(obj);
VALUE val;
if (rb_st_lookup(iv_table, (st_data_t)id, (st_data_t *)&val)) {
return val;
@@ -1408,7 +1385,7 @@ rb_ivar_lookup(VALUE obj, ID id, VALUE undef)
}
RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj));
- ivar_list = rb_imemo_class_fields_ptr(obj);
+ ivar_list = rb_imemo_fields_ptr(obj);
break;
}
case T_OBJECT:
@@ -1432,19 +1409,21 @@ rb_ivar_lookup(VALUE obj, ID id, VALUE undef)
default:
shape_id = RBASIC_SHAPE_ID(obj);
if (rb_obj_exivar_p(obj)) {
- struct gen_fields_tbl *fields_tbl;
- rb_gen_fields_tbl_get(obj, id, &fields_tbl);
+ VALUE fields_obj = 0;
+ rb_gen_fields_tbl_get(obj, id, &fields_obj);
- if (rb_shape_obj_too_complex_p(obj)) {
+ RUBY_ASSERT(fields_obj);
+
+ if (rb_shape_obj_too_complex_p(fields_obj)) {
VALUE val;
- if (rb_st_lookup(fields_tbl->as.complex.table, (st_data_t)id, (st_data_t *)&val)) {
+ if (rb_st_lookup(rb_imemo_fields_complex_tbl(fields_obj), (st_data_t)id, (st_data_t *)&val)) {
return val;
}
else {
return undef;
}
}
- ivar_list = fields_tbl->as.shape.fields;
+ ivar_list = rb_imemo_fields_ptr(fields_obj);
}
else {
return undef;
@@ -1486,7 +1465,7 @@ rb_ivar_delete(VALUE obj, ID id, VALUE undef)
VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
if (fields_obj) {
if (rb_multi_ractor_p()) {
- fields_obj = rb_imemo_class_fields_clone(fields_obj);
+ fields_obj = rb_imemo_fields_clone(fields_obj);
val = rb_ivar_delete(fields_obj, id, undef);
RCLASS_WRITABLE_SET_FIELDS_OBJ(obj, fields_obj);
}
@@ -1523,16 +1502,16 @@ rb_ivar_delete(VALUE obj, ID id, VALUE undef)
rb_bug("Unreachable");
break;
case T_IMEMO:
- RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_class_fields));
- fields = rb_imemo_class_fields_ptr(obj);
+ RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields));
+ fields = rb_imemo_fields_ptr(obj);
break;
case T_OBJECT:
fields = ROBJECT_FIELDS(obj);
break;
default: {
- struct gen_fields_tbl *fields_tbl;
- rb_gen_fields_tbl_get(obj, id, &fields_tbl);
- fields = fields_tbl->as.shape.fields;
+ VALUE fields_obj;
+ rb_gen_fields_tbl_get(obj, id, &fields_obj);
+ fields = rb_imemo_fields_ptr(fields_obj);
break;
}
}
@@ -1576,8 +1555,8 @@ too_complex:
break;
case T_IMEMO:
- RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_class_fields));
- table = rb_imemo_class_fields_complex_tbl(obj);
+ RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields));
+ table = rb_imemo_fields_complex_tbl(obj);
break;
case T_OBJECT:
@@ -1585,9 +1564,9 @@ too_complex:
break;
default: {
- struct gen_fields_tbl *fields_tbl;
- if (rb_gen_fields_tbl_get(obj, 0, &fields_tbl)) {
- table = fields_tbl->as.complex.table;
+ VALUE fields_obj;
+ if (rb_gen_fields_tbl_get(obj, 0, &fields_obj)) {
+ table = rb_imemo_fields_complex_tbl(fields_obj);
}
break;
}
@@ -1609,6 +1588,8 @@ rb_attr_delete(VALUE obj, ID id)
return rb_ivar_delete(obj, id, Qnil);
}
+static inline void generic_update_fields_obj(VALUE obj, VALUE fields_obj, const VALUE original_fields_obj);
+
static shape_id_t
obj_transition_too_complex(VALUE obj, st_table *table)
{
@@ -1619,46 +1600,37 @@ obj_transition_too_complex(VALUE obj, st_table *table)
RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj));
shape_id_t shape_id = rb_shape_transition_complex(obj);
- VALUE *old_fields = NULL;
-
switch (BUILTIN_TYPE(obj)) {
case T_OBJECT:
- if (!(RBASIC(obj)->flags & ROBJECT_EMBED)) {
- old_fields = ROBJECT_FIELDS(obj);
+ {
+ VALUE *old_fields = NULL;
+ if (!(RBASIC(obj)->flags & ROBJECT_EMBED)) {
+ old_fields = ROBJECT_FIELDS(obj);
+ }
+ RBASIC_SET_SHAPE_ID(obj, shape_id);
+ ROBJECT_SET_FIELDS_HASH(obj, table);
+ if (old_fields) {
+ xfree(old_fields);
+ }
}
- RBASIC_SET_SHAPE_ID(obj, shape_id);
- ROBJECT_SET_FIELDS_HASH(obj, table);
break;
case T_CLASS:
case T_MODULE:
rb_bug("Unreachable");
break;
default:
- RB_VM_LOCKING() {
- struct st_table *gen_ivs = generic_fields_tbl_no_ractor_check(obj);
-
- struct gen_fields_tbl *old_fields_tbl = NULL;
- st_lookup(gen_ivs, (st_data_t)obj, (st_data_t *)&old_fields_tbl);
-
- if (old_fields_tbl) {
- /* We need to modify old_fields_tbl to have the too complex shape
- * and hold the table because the xmalloc could trigger a GC
- * compaction. We want the table to be updated rather than
- * the original fields. */
- rb_obj_set_shape_id(obj, shape_id);
- old_fields_tbl->as.complex.table = table;
- old_fields = (VALUE *)old_fields_tbl;
- }
-
- struct gen_fields_tbl *fields_tbl = xmalloc(sizeof(struct gen_fields_tbl));
- fields_tbl->as.complex.table = table;
- st_insert(gen_ivs, (st_data_t)obj, (st_data_t)fields_tbl);
+ {
+ VALUE fields_obj = rb_imemo_fields_new_complex_tbl(rb_obj_class(obj), table);
+ RBASIC_SET_SHAPE_ID(fields_obj, shape_id);
+ RB_VM_LOCKING() {
+ const VALUE original_fields_obj = generic_fields_lookup(obj, 0, false);
+ generic_update_fields_obj(obj, fields_obj, original_fields_obj);
+ }
RBASIC_SET_SHAPE_ID(obj, shape_id);
}
}
- xfree(old_fields);
return shape_id;
}
@@ -1673,12 +1645,12 @@ rb_obj_init_too_complex(VALUE obj, st_table *table)
obj_transition_too_complex(obj, table);
}
+void rb_obj_copy_fields_to_hash_table(VALUE obj, st_table *table);
+
// Copy all object fields, including ivars and internal object_id, etc
shape_id_t
rb_evict_fields_to_hash(VALUE obj)
{
- void rb_obj_copy_fields_to_hash_table(VALUE obj, st_table *table);
-
RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj));
st_table *table = st_init_numtable_with_size(RSHAPE_LEN(RBASIC_SHAPE_ID(obj)));
@@ -1809,134 +1781,174 @@ general_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val, void *data,
}
}
-struct gen_fields_lookup_ensure_size {
- VALUE obj;
- ID id;
- shape_id_t shape_id;
- bool resize;
-};
+static inline void
+generic_update_fields_obj(VALUE obj, VALUE fields_obj, const VALUE original_fields_obj)
+{
+ if (fields_obj != original_fields_obj) {
+ if (original_fields_obj) {
+ // Clear root shape to avoid triggering cleanup such as free_object_id.
+ rb_imemo_fields_clear(original_fields_obj);
+ }
-static VALUE *
-generic_ivar_set_shape_fields(VALUE obj, void *data)
+ generic_fields_insert(obj, fields_obj);
+ }
+}
+
+static void
+generic_ivar_set(VALUE obj, ID id, VALUE val)
{
- RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj));
+ bool existing = true;
- struct gen_fields_lookup_ensure_size *fields_lookup = data;
- struct gen_fields_tbl *fields_tbl = NULL;
+ VALUE fields_obj = generic_fields_lookup(obj, id, false);
- // We can't use st_update, since when resizing the fields table GC can
- // happen, which will modify the st_table and may rebuild it
- RB_VM_LOCKING() {
- st_table *tbl = generic_fields_tbl(obj, fields_lookup->id, false);
- int existing = st_lookup(tbl, (st_data_t)obj, (st_data_t *)&fields_tbl);
-
- if (!existing || fields_lookup->resize) {
- uint32_t new_capa = RSHAPE_CAPACITY(fields_lookup->shape_id);
- uint32_t old_capa = RSHAPE_CAPACITY(RSHAPE_PARENT(fields_lookup->shape_id));
-
- if (existing) {
- RUBY_ASSERT(RSHAPE_TYPE_P(fields_lookup->shape_id, SHAPE_IVAR) || RSHAPE_TYPE_P(fields_lookup->shape_id, SHAPE_OBJ_ID));
- RUBY_ASSERT(old_capa < new_capa);
- RUBY_ASSERT(fields_tbl);
- } else {
- RUBY_ASSERT(!fields_tbl);
- RUBY_ASSERT(old_capa == 0);
- }
- RUBY_ASSERT(new_capa > 0);
+ const VALUE original_fields_obj = fields_obj;
+ if (!fields_obj) {
+ fields_obj = rb_imemo_fields_new(rb_obj_class(obj), 1);
+ }
+ RUBY_ASSERT(RBASIC_SHAPE_ID(obj) == RBASIC_SHAPE_ID(fields_obj));
- struct gen_fields_tbl *old_fields_tbl = fields_tbl;
- fields_tbl = xmalloc(gen_fields_tbl_bytes(new_capa));
- if (old_fields_tbl) {
- memcpy(fields_tbl, old_fields_tbl, gen_fields_tbl_bytes(old_capa));
- }
- st_insert(tbl, (st_data_t)obj, (st_data_t)fields_tbl);
- if (old_fields_tbl) {
- xfree(old_fields_tbl);
+ shape_id_t current_shape_id = RBASIC_SHAPE_ID(fields_obj);
+ shape_id_t next_shape_id = current_shape_id;
+
+ if (UNLIKELY(rb_shape_too_complex_p(current_shape_id))) {
+ goto too_complex;
+ }
+
+ attr_index_t index;
+ if (!rb_shape_get_iv_index(current_shape_id, id, &index)) {
+ existing = false;
+
+ index = RSHAPE_LEN(current_shape_id);
+ if (index >= SHAPE_MAX_FIELDS) {
+ rb_raise(rb_eArgError, "too many instance variables");
+ }
+
+ next_shape_id = rb_shape_transition_add_ivar(fields_obj, id);
+ if (UNLIKELY(rb_shape_too_complex_p(next_shape_id))) {
+ attr_index_t current_len = RSHAPE_LEN(current_shape_id);
+ fields_obj = rb_imemo_fields_new_complex(rb_obj_class(obj), current_len + 1);
+ if (current_len) {
+ rb_obj_copy_fields_to_hash_table(original_fields_obj, rb_imemo_fields_complex_tbl(fields_obj));
}
+ RBASIC_SET_SHAPE_ID(fields_obj, next_shape_id);
+ goto too_complex;
}
- if (fields_lookup->shape_id) {
- rb_obj_set_shape_id(fields_lookup->obj, fields_lookup->shape_id);
+ attr_index_t next_capacity = RSHAPE_CAPACITY(next_shape_id);
+ attr_index_t current_capacity = RSHAPE_CAPACITY(current_shape_id);
+
+ if (next_capacity != current_capacity) {
+ RUBY_ASSERT(next_capacity > current_capacity);
+
+ fields_obj = rb_imemo_fields_new(rb_obj_class(obj), next_capacity);
+ if (original_fields_obj) {
+ attr_index_t fields_count = RSHAPE_LEN(current_shape_id);
+ VALUE *fields = rb_imemo_fields_ptr(fields_obj);
+ MEMCPY(fields, rb_imemo_fields_ptr(original_fields_obj), VALUE, fields_count);
+ for (attr_index_t i = 0; i < fields_count; i++) {
+ RB_OBJ_WRITTEN(fields_obj, Qundef, fields[i]);
+ }
+ }
}
+
+ RUBY_ASSERT(RSHAPE(next_shape_id)->type == SHAPE_IVAR);
+ RUBY_ASSERT(index == (RSHAPE_LEN(next_shape_id) - 1));
}
- return fields_tbl->as.shape.fields;
-}
+ VALUE *fields = rb_imemo_fields_ptr(fields_obj);
+ RB_OBJ_WRITE(fields_obj, &fields[index], val);
-static void
-generic_ivar_set_shape_resize_fields(VALUE obj, attr_index_t _old_capa, attr_index_t new_capa, void *data)
-{
- struct gen_fields_lookup_ensure_size *fields_lookup = data;
+ if (!existing) {
+ RBASIC_SET_SHAPE_ID(fields_obj, next_shape_id);
+ }
- fields_lookup->resize = true;
-}
+ generic_update_fields_obj(obj, fields_obj, original_fields_obj);
-static void
-generic_ivar_set_set_shape_id(VALUE obj, shape_id_t shape_id, void *data)
-{
- struct gen_fields_lookup_ensure_size *fields_lookup = data;
+ if (!existing) {
+ RBASIC_SET_SHAPE_ID(obj, next_shape_id);
+ }
- fields_lookup->shape_id = shape_id;
-}
+ RUBY_ASSERT(RBASIC_SHAPE_ID(obj) == RBASIC_SHAPE_ID(fields_obj));
-static shape_id_t
-generic_ivar_set_transition_too_complex(VALUE obj, void *_data)
-{
- shape_id_t new_shape_id = rb_evict_fields_to_hash(obj);
- return new_shape_id;
-}
+ return;
-static st_table *
-generic_ivar_set_too_complex_table(VALUE obj, void *data)
-{
- struct gen_fields_lookup_ensure_size *fields_lookup = data;
+too_complex:
+ {
+ st_table *table = rb_imemo_fields_complex_tbl(fields_obj);
+ existing = st_insert(table, (st_data_t)id, (st_data_t)val);
+ RB_OBJ_WRITTEN(fields_obj, Qundef, val);
- struct gen_fields_tbl *fields_tbl;
- if (!rb_gen_fields_tbl_get(obj, 0, &fields_tbl)) {
- fields_tbl = xmalloc(sizeof(struct gen_fields_tbl));
- fields_tbl->as.complex.table = st_init_numtable_with_size(1);
+ generic_update_fields_obj(obj, fields_obj, original_fields_obj);
- RB_VM_LOCKING() {
- st_insert(generic_fields_tbl(obj, fields_lookup->id, false), (st_data_t)obj, (st_data_t)fields_tbl);
+ if (!existing) {
+ RBASIC_SET_SHAPE_ID(obj, next_shape_id);
}
}
- RUBY_ASSERT(rb_shape_obj_too_complex_p(obj));
+ RUBY_ASSERT(RBASIC_SHAPE_ID(obj) == RBASIC_SHAPE_ID(fields_obj));
- return fields_tbl->as.complex.table;
+ return;
}
static void
-generic_ivar_set(VALUE obj, ID id, VALUE val)
+generic_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val)
{
- struct gen_fields_lookup_ensure_size fields_lookup = {
- .obj = obj,
- .id = id,
- .resize = false,
- };
+ bool existing = true;
- general_ivar_set(obj, id, val, &fields_lookup,
- generic_ivar_set_shape_fields,
- generic_ivar_set_shape_resize_fields,
- generic_ivar_set_set_shape_id,
- generic_ivar_set_transition_too_complex,
- generic_ivar_set_too_complex_table);
-}
+ VALUE fields_obj = generic_fields_lookup(obj, RSHAPE_EDGE_NAME(target_shape_id), false);
+ const VALUE original_fields_obj = fields_obj;
-static void
-generic_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val)
-{
- struct gen_fields_lookup_ensure_size fields_lookup = {
- .obj = obj,
- .resize = false,
- };
+ shape_id_t current_shape_id = fields_obj ? RBASIC_SHAPE_ID(fields_obj) : ROOT_SHAPE_ID;
+
+ if (UNLIKELY(rb_shape_too_complex_p(target_shape_id))) {
+ if (UNLIKELY(!rb_shape_too_complex_p(current_shape_id))) {
+ attr_index_t current_len = RSHAPE_LEN(current_shape_id);
+ fields_obj = rb_imemo_fields_new_complex(rb_obj_class(obj), current_len + 1);
+ if (current_len) {
+ rb_obj_copy_fields_to_hash_table(original_fields_obj, rb_imemo_fields_complex_tbl(fields_obj));
+ }
+
+ current_shape_id = target_shape_id;
+ }
+
+ existing = false;
+ st_table *table = rb_imemo_fields_complex_tbl(fields_obj);
+
+ RUBY_ASSERT(RSHAPE_EDGE_NAME(target_shape_id));
+ st_insert(table, (st_data_t)RSHAPE_EDGE_NAME(target_shape_id), (st_data_t)val);
+ RB_OBJ_WRITTEN(fields_obj, Qundef, val);
+ RBASIC_SET_SHAPE_ID(fields_obj, target_shape_id);
+ }
+ else {
+ attr_index_t index = RSHAPE_INDEX(target_shape_id);
+ if (index >= RSHAPE_CAPACITY(current_shape_id)) {
+ fields_obj = rb_imemo_fields_new(rb_obj_class(obj), index);
+ if (original_fields_obj) {
+ attr_index_t fields_count = RSHAPE_LEN(current_shape_id);
+ VALUE *fields = rb_imemo_fields_ptr(fields_obj);
+ MEMCPY(fields, rb_imemo_fields_ptr(original_fields_obj), VALUE, fields_count);
+ for (attr_index_t i = 0; i < fields_count; i++) {
+ RB_OBJ_WRITTEN(fields_obj, Qundef, fields[i]);
+ }
+ }
+ }
+
+ VALUE *table = rb_imemo_fields_ptr(fields_obj);
+ RB_OBJ_WRITE(fields_obj, &table[index], val);
+
+ if (RSHAPE_LEN(target_shape_id) > RSHAPE_LEN(current_shape_id)) {
+ existing = false;
+ RBASIC_SET_SHAPE_ID(fields_obj, target_shape_id);
+ }
+ }
+
+ generic_update_fields_obj(obj, fields_obj, original_fields_obj);
+
+ if (!existing) {
+ RBASIC_SET_SHAPE_ID(obj, target_shape_id);
+ }
- general_field_set(obj, target_shape_id, val, &fields_lookup,
- generic_ivar_set_shape_fields,
- generic_ivar_set_shape_resize_fields,
- generic_ivar_set_set_shape_id,
- generic_ivar_set_transition_too_complex,
- generic_ivar_set_too_complex_table);
+ RUBY_ASSERT(RBASIC_SHAPE_ID(obj) == RBASIC_SHAPE_ID(fields_obj));
}
void
@@ -2155,8 +2167,8 @@ ivar_defined0(VALUE obj, ID id)
break;
case T_IMEMO:
- RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_class_fields));
- table = rb_imemo_class_fields_complex_tbl(obj);
+ RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields));
+ table = rb_imemo_fields_complex_tbl(obj);
break;
case T_OBJECT:
@@ -2164,11 +2176,10 @@ ivar_defined0(VALUE obj, ID id)
break;
default: {
- struct gen_fields_tbl *fields_tbl;
- if (rb_gen_fields_tbl_get(obj, 0, &fields_tbl)) {
- table = fields_tbl->as.complex.table;
+ VALUE fields_obj;
+ if (rb_gen_fields_tbl_get(obj, 0, &fields_obj)) {
+ table = rb_imemo_fields_complex_tbl(fields_obj);
}
- break;
}
}
@@ -2224,27 +2235,23 @@ iterate_over_shapes_callback(shape_id_t shape_id, void *data)
return ST_CONTINUE;
}
- VALUE *iv_list;
+ VALUE *fields;
switch (BUILTIN_TYPE(itr_data->obj)) {
case T_OBJECT:
RUBY_ASSERT(!rb_shape_obj_too_complex_p(itr_data->obj));
- iv_list = ROBJECT_FIELDS(itr_data->obj);
+ fields = ROBJECT_FIELDS(itr_data->obj);
break;
- case T_CLASS:
- case T_MODULE:
- rb_bug("Unreachable");
case T_IMEMO:
- RUBY_ASSERT(IMEMO_TYPE_P(itr_data->obj, imemo_class_fields));
+ RUBY_ASSERT(IMEMO_TYPE_P(itr_data->obj, imemo_fields));
RUBY_ASSERT(!rb_shape_obj_too_complex_p(itr_data->obj));
- iv_list = rb_imemo_class_fields_ptr(itr_data->obj);
+ fields = rb_imemo_fields_ptr(itr_data->obj);
break;
default:
- iv_list = itr_data->fields_tbl->as.shape.fields;
- break;
+ rb_bug("Unreachable");
}
- VALUE val = iv_list[RSHAPE_INDEX(shape_id)];
+ VALUE val = fields[RSHAPE_INDEX(shape_id)];
return itr_data->func(RSHAPE_EDGE_NAME(shape_id), val, itr_data->arg);
}
@@ -2286,33 +2293,9 @@ obj_fields_each(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg, b
}
static void
-gen_fields_each(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg, bool ivar_only)
-{
- struct gen_fields_tbl *fields_tbl;
- if (!rb_gen_fields_tbl_get(obj, 0, &fields_tbl)) return;
-
- struct iv_itr_data itr_data = {
- .obj = obj,
- .fields_tbl = fields_tbl,
- .arg = arg,
- .func = func,
- .ivar_only = ivar_only,
- };
-
- shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
- if (rb_shape_too_complex_p(shape_id)) {
- rb_st_foreach(fields_tbl->as.complex.table, each_hash_iv, (st_data_t)&itr_data);
- }
- else {
- itr_data.fields = fields_tbl->as.shape.fields;
- iterate_over_shapes(shape_id, func, &itr_data);
- }
-}
-
-static void
-class_fields_each(VALUE fields_obj, rb_ivar_foreach_callback_func *func, st_data_t arg, bool ivar_only)
+imemo_fields_each(VALUE fields_obj, rb_ivar_foreach_callback_func *func, st_data_t arg, bool ivar_only)
{
- IMEMO_TYPE_P(fields_obj, imemo_class_fields);
+ IMEMO_TYPE_P(fields_obj, imemo_fields);
struct iv_itr_data itr_data = {
.obj = fields_obj,
@@ -2323,10 +2306,10 @@ class_fields_each(VALUE fields_obj, rb_ivar_foreach_callback_func *func, st_data
shape_id_t shape_id = RBASIC_SHAPE_ID(fields_obj);
if (rb_shape_too_complex_p(shape_id)) {
- rb_st_foreach(rb_imemo_class_fields_complex_tbl(fields_obj), each_hash_iv, (st_data_t)&itr_data);
+ rb_st_foreach(rb_imemo_fields_complex_tbl(fields_obj), each_hash_iv, (st_data_t)&itr_data);
}
else {
- itr_data.fields = rb_imemo_class_fields_ptr(fields_obj);
+ itr_data.fields = rb_imemo_fields_ptr(fields_obj);
iterate_over_shapes(shape_id, func, &itr_data);
}
}
@@ -2334,8 +2317,8 @@ class_fields_each(VALUE fields_obj, rb_ivar_foreach_callback_func *func, st_data
void
rb_copy_generic_ivar(VALUE dest, VALUE obj)
{
- struct gen_fields_tbl *obj_fields_tbl;
- struct gen_fields_tbl *new_fields_tbl;
+ VALUE fields_obj;
+ VALUE new_fields_obj;
rb_check_frozen(dest);
@@ -2343,19 +2326,16 @@ rb_copy_generic_ivar(VALUE dest, VALUE obj)
return;
}
- unsigned long src_num_ivs = rb_ivar_count(obj);
- if (!src_num_ivs) {
- goto clear;
- }
-
shape_id_t src_shape_id = rb_obj_shape_id(obj);
- if (rb_gen_fields_tbl_get(obj, 0, &obj_fields_tbl)) {
- if (gen_fields_tbl_count(obj, obj_fields_tbl) == 0)
+ if (rb_gen_fields_tbl_get(obj, 0, &fields_obj)) {
+ unsigned long src_num_ivs = rb_ivar_count(fields_obj);
+ if (!src_num_ivs) {
goto clear;
+ }
if (rb_shape_too_complex_p(src_shape_id)) {
- rb_shape_copy_complex_ivars(dest, obj, src_shape_id, obj_fields_tbl->as.complex.table);
+ rb_shape_copy_complex_ivars(dest, obj, src_shape_id, rb_imemo_fields_complex_tbl(fields_obj));
return;
}
@@ -2370,7 +2350,6 @@ rb_copy_generic_ivar(VALUE dest, VALUE obj)
st_table *table = rb_st_init_numtable_with_size(src_num_ivs);
rb_obj_copy_ivs_to_hash_table(obj, table);
rb_obj_init_too_complex(dest, table);
-
return;
}
}
@@ -2380,25 +2359,19 @@ rb_copy_generic_ivar(VALUE dest, VALUE obj)
return;
}
- uint32_t dest_capa = RSHAPE_CAPACITY(dest_shape_id);
- RUBY_ASSERT(dest_capa > 0);
- new_fields_tbl = xmalloc(gen_fields_tbl_bytes(dest_capa));
-
- VALUE *src_buf = obj_fields_tbl->as.shape.fields;
- VALUE *dest_buf = new_fields_tbl->as.shape.fields;
-
+ new_fields_obj = rb_imemo_fields_new(rb_obj_class(dest), RSHAPE_CAPACITY(dest_shape_id));
+ VALUE *src_buf = rb_imemo_fields_ptr(fields_obj);
+ VALUE *dest_buf = rb_imemo_fields_ptr(new_fields_obj);
rb_shape_copy_fields(dest, dest_buf, dest_shape_id, obj, src_buf, src_shape_id);
+ RBASIC_SET_SHAPE_ID(new_fields_obj, dest_shape_id);
- /*
- * c.fields_tbl may change in gen_fields_copy due to realloc,
- * no need to free
- */
RB_VM_LOCKING() {
generic_fields_tbl_no_ractor_check(dest);
- st_insert(generic_fields_tbl_no_ractor_check(obj), (st_data_t)dest, (st_data_t)new_fields_tbl);
+ st_insert(generic_fields_tbl_no_ractor_check(obj), (st_data_t)dest, (st_data_t)new_fields_obj);
+ RB_OBJ_WRITTEN(dest, Qundef, new_fields_obj);
}
- rb_obj_set_shape_id(dest, dest_shape_id);
+ RBASIC_SET_SHAPE_ID(dest, dest_shape_id);
}
return;
@@ -2426,8 +2399,8 @@ rb_field_foreach(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg,
if (SPECIAL_CONST_P(obj)) return;
switch (BUILTIN_TYPE(obj)) {
case T_IMEMO:
- if (IMEMO_TYPE_P(obj, imemo_class_fields)) {
- class_fields_each(obj, func, arg, ivar_only);
+ if (IMEMO_TYPE_P(obj, imemo_fields)) {
+ imemo_fields_each(obj, func, arg, ivar_only);
}
break;
case T_OBJECT:
@@ -2439,13 +2412,16 @@ rb_field_foreach(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg,
IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(0);
VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
if (fields_obj) {
- class_fields_each(fields_obj, func, arg, ivar_only);
+ imemo_fields_each(fields_obj, func, arg, ivar_only);
}
}
break;
default:
if (rb_obj_exivar_p(obj)) {
- gen_fields_each(obj, func, arg, ivar_only);
+ VALUE fields_obj = 0;
+ if (!rb_gen_fields_tbl_get(obj, 0, &fields_obj)) return;
+
+ imemo_fields_each(fields_obj, func, arg, ivar_only);
}
break;
}
@@ -2467,6 +2443,7 @@ rb_ivar_count(VALUE obj)
case T_OBJECT:
iv_count = ROBJECT_FIELDS_COUNT(obj);
break;
+
case T_CLASS:
case T_MODULE:
{
@@ -2475,16 +2452,37 @@ rb_ivar_count(VALUE obj)
return 0;
}
if (rb_shape_obj_too_complex_p(fields_obj)) {
- return rb_st_table_size(rb_imemo_class_fields_complex_tbl(fields_obj));
+ iv_count = rb_st_table_size(rb_imemo_fields_complex_tbl(fields_obj));
+ }
+ else {
+ iv_count = RBASIC_FIELDS_COUNT(fields_obj);
}
- return RBASIC_FIELDS_COUNT(fields_obj);
}
+ break;
+
+ case T_IMEMO:
+ RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields));
+
+ if (rb_shape_obj_too_complex_p(obj)) {
+ iv_count = rb_st_table_size(rb_imemo_fields_complex_tbl(obj));
+ }
+ else {
+ iv_count = RBASIC_FIELDS_COUNT(obj);
+ }
+ break;
+
default:
if (rb_obj_exivar_p(obj)) {
- struct gen_fields_tbl *fields_tbl;
- if (rb_gen_fields_tbl_get(obj, 0, &fields_tbl)) {
- iv_count = gen_fields_tbl_count(obj, fields_tbl);
+ if (rb_shape_obj_too_complex_p(obj)) {
+ VALUE fields_obj;
+
+ if (rb_gen_fields_tbl_get(obj, 0, &fields_obj)) {
+ iv_count = rb_st_table_size(rb_imemo_fields_complex_tbl(fields_obj));
+ }
+ }
+ else {
+ iv_count = RBASIC_FIELDS_COUNT(obj);
}
}
break;
@@ -4689,7 +4687,7 @@ class_fields_ivar_set(VALUE klass, VALUE fields_obj, ID id, VALUE val, bool conc
{
bool existing = true;
const VALUE original_fields_obj = fields_obj;
- fields_obj = original_fields_obj ? original_fields_obj : rb_imemo_class_fields_new(klass, 1);
+ fields_obj = original_fields_obj ? original_fields_obj : rb_imemo_fields_new(rb_singleton_class(klass), 1);
shape_id_t current_shape_id = RBASIC_SHAPE_ID(fields_obj);
shape_id_t next_shape_id = current_shape_id;
@@ -4710,9 +4708,9 @@ class_fields_ivar_set(VALUE klass, VALUE fields_obj, ID id, VALUE val, bool conc
next_shape_id = rb_shape_transition_add_ivar(fields_obj, id);
if (UNLIKELY(rb_shape_too_complex_p(next_shape_id))) {
attr_index_t current_len = RSHAPE_LEN(current_shape_id);
- fields_obj = rb_imemo_class_fields_new_complex(klass, current_len + 1);
+ fields_obj = rb_imemo_fields_new_complex(rb_singleton_class(klass), current_len + 1);
if (current_len) {
- rb_obj_copy_fields_to_hash_table(original_fields_obj, rb_imemo_class_fields_complex_tbl(fields_obj));
+ rb_obj_copy_fields_to_hash_table(original_fields_obj, rb_imemo_fields_complex_tbl(fields_obj));
RBASIC_SET_SHAPE_ID(fields_obj, next_shape_id);
}
goto too_complex;
@@ -4726,9 +4724,9 @@ class_fields_ivar_set(VALUE klass, VALUE fields_obj, ID id, VALUE val, bool conc
// We allocate a new fields_obj even when concurrency isn't a concern
// so that we're embedded as long as possible.
- fields_obj = rb_imemo_class_fields_new(klass, next_capacity);
+ fields_obj = rb_imemo_fields_new(rb_singleton_class(klass), next_capacity);
if (original_fields_obj) {
- MEMCPY(rb_imemo_class_fields_ptr(fields_obj), rb_imemo_class_fields_ptr(original_fields_obj), VALUE, RSHAPE_LEN(current_shape_id));
+ MEMCPY(rb_imemo_fields_ptr(fields_obj), rb_imemo_fields_ptr(original_fields_obj), VALUE, RSHAPE_LEN(current_shape_id));
}
}
@@ -4736,7 +4734,7 @@ class_fields_ivar_set(VALUE klass, VALUE fields_obj, ID id, VALUE val, bool conc
RUBY_ASSERT(index == (RSHAPE_LEN(next_shape_id) - 1));
}
- VALUE *fields = rb_imemo_class_fields_ptr(fields_obj);
+ VALUE *fields = rb_imemo_fields_ptr(fields_obj);
RB_OBJ_WRITE(fields_obj, &fields[index], val);
if (!existing) {
@@ -4748,7 +4746,7 @@ class_fields_ivar_set(VALUE klass, VALUE fields_obj, ID id, VALUE val, bool conc
too_complex:
{
- st_table *table = rb_imemo_class_fields_complex_tbl(fields_obj);
+ st_table *table = rb_imemo_fields_complex_tbl(fields_obj);
existing = st_insert(table, (st_data_t)id, (st_data_t)val);
RB_OBJ_WRITTEN(fields_obj, Qundef, val);
@@ -4776,13 +4774,14 @@ rb_class_ivar_set(VALUE obj, ID id, VALUE val)
if (new_fields_obj != original_fields_obj) {
RCLASS_WRITABLE_SET_FIELDS_OBJ(obj, new_fields_obj);
-
- // TODO: What should we set as the T_CLASS shape_id?
- // In most case we can replicate the single `fields_obj` shape
- // but in namespaced case?
- // Perhaps INVALID_SHAPE_ID?
- RBASIC_SET_SHAPE_ID(obj, RBASIC_SHAPE_ID(new_fields_obj));
}
+
+ // TODO: What should we set as the T_CLASS shape_id?
+ // In most case we can replicate the single `fields_obj` shape
+ // but in namespaced case?
+ // Perhaps INVALID_SHAPE_ID?
+ RBASIC_SET_SHAPE_ID(obj, RBASIC_SHAPE_ID(new_fields_obj));
+
return existing;
}