X-Git-Url: https://repo.or.cz/ruby.git/blobdiff_plain/fba647087b1d61a890537cf9294e4d2074f7572b..7f0c6d30d3d42a9d9ee9ab79e2acd86baa9184f4:/ruby_parser.c

diff --git a/ruby_parser.c b/ruby_parser.c
index 74e02996b8..267f619bf9 100644
--- a/ruby_parser.c
+++ b/ruby_parser.c
@@ -1,5 +1,7 @@
 /* This is a wrapper for parse.y */
 
+#include "internal/parse.h"
+#include "internal/re.h"
 #include "internal/ruby_parser.h"
 
 #include "node.h"
@@ -17,7 +19,6 @@
 #include "internal/gc.h"
 #include "internal/hash.h"
 #include "internal/io.h"
-#include "internal/parse.h"
 #include "internal/rational.h"
 #include "internal/re.h"
 #include "internal/string.h"
@@ -31,53 +32,14 @@
 #include "vm_core.h"
 #include "symbol.h"
 
-struct ruby_parser {
-    rb_parser_t *parser_params;
-};
-
-static void
-parser_mark(void *ptr)
-{
-    struct ruby_parser *parser = (struct ruby_parser*)ptr;
-    rb_ruby_parser_mark(parser->parser_params);
-}
-
-static void
-parser_free(void *ptr)
-{
-    struct ruby_parser *parser = (struct ruby_parser*)ptr;
-    rb_ruby_parser_free(parser->parser_params);
-}
-
-static size_t
-parser_memsize(const void *ptr)
-{
-    struct ruby_parser *parser = (struct ruby_parser*)ptr;
-    return rb_ruby_parser_memsize(parser->parser_params);
-}
-
-static const rb_data_type_t ruby_parser_data_type = {
-    "parser",
-    {
-        parser_mark,
-        parser_free,
-        parser_memsize,
-    },
-    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
-};
-
-static int
-is_ascii_string2(VALUE str)
-{
-    return is_ascii_string(str);
-}
+#define parser_encoding const void
 
 RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 6, 0)
 static VALUE
 syntax_error_append(VALUE exc, VALUE file, int line, int column,
-                       void *enc, const char *fmt, va_list args)
+                    parser_encoding *enc, const char *fmt, va_list args)
 {
-    return rb_syntax_error_append(exc, file, line, column, (rb_encoding *)enc, fmt, args);
+    return rb_syntax_error_append(exc, file, line, column, enc, fmt, args);
 }
 
 static int
@@ -92,36 +54,10 @@ dvar_defined(ID id, const void *p)
     return rb_dvar_defined(id, (const rb_iseq_t *)p);
 }
 
-static bool
-hash_literal_key_p(VALUE k)
-{
-    switch (OBJ_BUILTIN_TYPE(k)) {
-      case T_NODE:
-        return false;
-      default:
-        return true;
-    }
-}
-
 static int
-literal_cmp(VALUE val, VALUE lit)
+is_usascii_enc(parser_encoding *enc)
 {
-    if (val == lit) return 0;
-    if (!hash_literal_key_p(val) || !hash_literal_key_p(lit)) return -1;
-    return rb_iseq_cdhash_cmp(val, lit);
-}
-
-static st_index_t
-literal_hash(VALUE a)
-{
-    if (!hash_literal_key_p(a)) return (st_index_t)a;
-    return rb_iseq_cdhash_hash(a);
-}
-
-static int
-is_usascii_enc(void *enc)
-{
-    return rb_is_usascii_enc((rb_encoding *)enc);
+    return rb_is_usascii_enc(enc);
 }
 
 static int
@@ -143,21 +79,21 @@ is_notop_id2(ID id)
 }
 
 static VALUE
-enc_str_new(const char *ptr, long len, void *enc)
+enc_str_new(const char *ptr, long len, parser_encoding *enc)
 {
-    return rb_enc_str_new(ptr, len, (rb_encoding *)enc);
+    return rb_enc_str_new(ptr, len, enc);
 }
 
 static int
-enc_isalnum(OnigCodePoint c, void *enc)
+enc_isalnum(OnigCodePoint c, parser_encoding *enc)
 {
-    return rb_enc_isalnum(c, (rb_encoding *)enc);
+    return rb_enc_isalnum(c, enc);
 }
 
 static int
-enc_precise_mbclen(const char *p, const char *e, void *enc)
+enc_precise_mbclen(const char *p, const char *e, parser_encoding *enc)
 {
-    return rb_enc_precise_mbclen(p, e, (rb_encoding *)enc);
+    return rb_enc_precise_mbclen(p, e, enc);
 }
 
 static int
@@ -173,123 +109,75 @@ mbclen_charfound_len(int len)
 }
 
 static const char *
-enc_name(void *enc)
+enc_name(parser_encoding *enc)
 {
-    return rb_enc_name((rb_encoding *)enc);
+    return rb_enc_name(enc);
 }
 
 static char *
-enc_prev_char(const char *s, const char *p, const char *e, void *enc)
+enc_prev_char(const char *s, const char *p, const char *e, parser_encoding *enc)
 {
-    return rb_enc_prev_char(s, p, e, (rb_encoding *)enc);
+    return rb_enc_prev_char(s, p, e, enc);
 }
 
-static void *
+static parser_encoding *
 enc_get(VALUE obj)
 {
-    return (void *)rb_enc_get(obj);
+    return rb_enc_get(obj);
 }
 
 static int
-enc_asciicompat(void *enc)
+enc_asciicompat(parser_encoding *enc)
 {
-    return rb_enc_asciicompat((rb_encoding *)enc);
+    return rb_enc_asciicompat(enc);
 }
 
-static void *
+static parser_encoding *
 utf8_encoding(void)
 {
-    return (void *)rb_utf8_encoding();
-}
-
-static VALUE
-enc_associate(VALUE obj, void *enc)
-{
-    return rb_enc_associate(obj, (rb_encoding *)enc);
+    return rb_utf8_encoding();
 }
 
-static void *
+static parser_encoding *
 ascii8bit_encoding(void)
 {
-    return (void *)rb_ascii8bit_encoding();
+    return rb_ascii8bit_encoding();
 }
 
 static int
-enc_codelen(int c, void *enc)
+enc_codelen(int c, parser_encoding *enc)
 {
-    return rb_enc_codelen(c, (rb_encoding *)enc);
-}
-
-static VALUE
-enc_str_buf_cat(VALUE str, const char *ptr, long len, void *enc)
-{
-    return rb_enc_str_buf_cat(str, ptr, len, (rb_encoding *)enc);
+    return rb_enc_codelen(c, enc);
 }
 
 static int
-enc_mbcput(unsigned int c, void *buf, void *enc)
+enc_mbcput(unsigned int c, void *buf, parser_encoding *enc)
 {
-    return rb_enc_mbcput(c, buf, (rb_encoding *)enc);
+    return rb_enc_mbcput(c, buf, enc);
 }
 
-static void *
+static parser_encoding *
 enc_from_index(int idx)
 {
-    return (void *)rb_enc_from_index(idx);
+    return rb_enc_from_index(idx);
 }
 
 static int
-enc_isspace(OnigCodePoint c, void *enc)
+enc_isspace(OnigCodePoint c, parser_encoding *enc)
 {
-    return rb_enc_isspace(c, (rb_encoding *)enc);
+    return rb_enc_isspace(c, enc);
 }
 
 static ID
-intern3(const char *name, long len, void *enc)
-{
-    return rb_intern3(name, len, (rb_encoding *)enc);
-}
-
-static void *
-enc_compatible(VALUE str1, VALUE str2)
-{
-    return (void *)rb_enc_compatible(str1, str2);
-}
-
-static VALUE
-enc_from_encoding(void *enc)
-{
-    return rb_enc_from_encoding((rb_encoding *)enc);
-}
-
-static int
-encoding_get(VALUE obj)
-{
-    return ENCODING_GET(obj);
-}
-
-static void
-encoding_set(VALUE obj, int encindex)
-{
-    ENCODING_SET(obj, encindex);
-}
-
-static int
-encoding_is_ascii8bit(VALUE obj)
+intern3(const char *name, long len, parser_encoding *enc)
 {
-    return ENCODING_IS_ASCII8BIT(obj);
-}
-
-static void *
-usascii_encoding(void)
-{
-    return (void *)rb_usascii_encoding();
+    return rb_intern3(name, len, enc);
 }
 
 static int
-enc_symname_type(const char *name, long len, void *enc, unsigned int allowed_attrset)
+enc_symname_type(const char *name, long len, parser_encoding *enc, unsigned int allowed_attrset)
 {
-    return rb_enc_symname_type(name, len, (rb_encoding *)enc, allowed_attrset);
+    return rb_enc_symname_type(name, len, enc, allowed_attrset);
 }
 
 typedef struct {
@@ -297,6 +185,7 @@ typedef struct {
     rb_encoding *enc;
     NODE *succ_block;
     const rb_code_location_t *loc;
+    rb_parser_assignable_func assignable;
 } reg_named_capture_assign_t;
 
 static int
@@ -310,11 +199,12 @@ reg_named_capture_assign_iter(const OnigUChar *name, const OnigUChar *name_end,
     long len = name_end - name;
     const char *s = (const char *)name;
 
-    return rb_reg_named_capture_assign_iter_impl(p, s, len, (void *)enc, &arg->succ_block, loc);
+    return rb_reg_named_capture_assign_iter_impl(p, s, len, enc, &arg->succ_block, loc, arg->assignable);
 }
 
 static NODE *
-reg_named_capture_assign(struct parser_params* p, VALUE regexp, const rb_code_location_t *loc)
+reg_named_capture_assign(struct parser_params* p, VALUE regexp, const rb_code_location_t *loc,
+                         rb_parser_assignable_func assignable)
 {
     reg_named_capture_assign_t arg;
 
@@ -322,24 +212,13 @@ reg_named_capture_assign(struct parser_params* p, VALUE regexp, const rb_code_lo
     arg.enc = rb_enc_get(regexp);
     arg.succ_block = 0;
     arg.loc = loc;
+    arg.assignable = assignable;
     onig_foreach_name(RREGEXP_PTR(regexp), reg_named_capture_assign_iter, &arg);
 
     if (!arg.succ_block) return 0;
     return RNODE_BLOCK(arg.succ_block)->nd_next;
 }
 
-static VALUE
-rbool(VALUE v)
-{
-    return RBOOL(v);
-}
-
-static int
-undef_p(VALUE v)
-{
-    return RB_UNDEF_P(v);
-}
-
 static int
 rtest(VALUE obj)
 {
@@ -358,24 +237,6 @@ syntax_error_new(void)
     return rb_class_new_instance(0, 0, rb_eSyntaxError);
 }
 
-static VALUE
-obj_write(VALUE old, VALUE *slot, VALUE young)
-{
-    return RB_OBJ_WRITE(old, slot, young);
-}
-
-static VALUE
-obj_written(VALUE old, VALUE slot, VALUE young)
-{
-    return RB_OBJ_WRITTEN(old, slot, young);
-}
-
-static VALUE
-default_rs(void)
-{
-    return rb_default_rs;
-}
-
 static void *
 memmove2(void *dest, const void *src, size_t t, size_t n)
 {
@@ -400,24 +261,6 @@ rb_errno_ptr2(void)
     return rb_errno_ptr();
 }
 
-static int
-type_p(VALUE obj, int t)
-{
-    return (int)RB_TYPE_P(obj, t);
-}
-
-static int
-fixnum_p(VALUE obj)
-{
-    return (int)RB_FIXNUM_P(obj);
-}
-
-static int
-symbol_p(VALUE obj)
-{
-    return (int)RB_SYMBOL_P(obj);
-}
-
 static void *
 zalloc(size_t elemsiz)
 {
@@ -430,12 +273,6 @@ gc_guard(VALUE obj)
     RB_GC_GUARD(obj);
 }
 
-static rb_imemo_tmpbuf_t *
-tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt)
-{
-    return rb_imemo_tmpbuf_parser_heap(buf, old_heap, cnt);
-}
-
 static VALUE
 arg_error(void)
 {
@@ -443,46 +280,39 @@ arg_error(void)
 }
 
 static VALUE
-ruby_vm_frozen_core(void)
+static_id2sym(ID id)
 {
-    return rb_mRubyVMFrozenCore;
+    return (((VALUE)(id)<<RUBY_SPECIAL_SHIFT)|SYMBOL_FLAG);
 }
 
-static int
-special_const_p(VALUE obj)
+static long
+str_coderange_scan_restartable(const char *s, const char *e, parser_encoding *enc, int *cr)
 {
-    return (int)RB_SPECIAL_CONST_P(obj);
+    return rb_str_coderange_scan_restartable(s, e, enc, cr);
 }
 
 static int
-builtin_type(VALUE obj)
+enc_mbminlen(parser_encoding *enc)
 {
-    return (int)RB_BUILTIN_TYPE(obj);
+    return rb_enc_mbminlen(enc);
 }
 
-static rb_ast_t *
-ast_new(VALUE nb)
+static bool
+enc_isascii(OnigCodePoint c, parser_encoding *enc)
 {
-    rb_ast_t *ast = (rb_ast_t *)rb_imemo_new(imemo_ast, 0, 0, 0, nb);
-    return ast;
+    return rb_enc_isascii(c, enc);
 }
 
-static VALUE
-static_id2sym(ID id)
+static OnigCodePoint
+enc_mbc_to_codepoint(const char *p, const char *e, parser_encoding *enc)
 {
-    return (((VALUE)(id)<<RUBY_SPECIAL_SHIFT)|SYMBOL_FLAG);
-}
+    const OnigUChar *up = RBIMPL_CAST((const OnigUChar *)p);
+    const OnigUChar *ue = RBIMPL_CAST((const OnigUChar *)e);
 
-static long
-str_coderange_scan_restartable(const char *s, const char *e, void *enc, int *cr)
-{
-    return rb_str_coderange_scan_restartable(s, e, (rb_encoding *)enc, cr);
+    return ONIGENC_MBC_TO_CODE((rb_encoding *)enc, up, ue);
 }
 
-VALUE rb_io_gets_internal(VALUE io);
 extern VALUE rb_eArgError;
-extern VALUE rb_mRubyVMFrozenCore;
-VALUE rb_node_case_when_optimizable_literal(const NODE *const node);
 
 static const rb_parser_config_t rb_global_parser_config = {
     .malloc = ruby_xmalloc,
@@ -497,31 +327,13 @@ static const rb_parser_config_t rb_global_parser_config = {
     .nonempty_memcpy = nonempty_memcpy,
     .xmalloc_mul_add = rb_xmalloc_mul_add,
 
-    .tmpbuf_parser_heap = tmpbuf_parser_heap,
-    .ast_new = ast_new,
-
     .compile_callback = rb_suppress_tracing,
     .reg_named_capture_assign = reg_named_capture_assign,
 
-    .obj_freeze = rb_obj_freeze,
-    .obj_hide = rb_obj_hide,
-    .type_p = type_p,
-    .obj_freeze_raw = OBJ_FREEZE_RAW,
-
-    .fixnum_p = fixnum_p,
-    .symbol_p = symbol_p,
-
     .attr_get = rb_attr_get,
 
-    .ary_new = rb_ary_new,
-    .ary_push = rb_ary_push,
     .ary_new_from_args = rb_ary_new_from_args,
     .ary_unshift = rb_ary_unshift,
-    .ary_new2 = rb_ary_new2,
-    .ary_clear = rb_ary_clear,
-    .ary_modify = rb_ary_modify,
-    .array_len = rb_array_len,
-    .array_aref = RARRAY_AREF,
 
     .make_temporary_id = rb_make_temporary_id,
     .is_local_id = is_local_id2,
@@ -538,51 +350,26 @@ static const rb_parser_config_t rb_global_parser_config = {
     .id2name = rb_id2name,
     .id2str = rb_id2str,
     .id2sym = rb_id2sym,
-    .sym2id = rb_sym2id,
 
     .str_catf = rb_str_catf,
     .str_cat_cstr = rb_str_cat_cstr,
-    .str_subseq = rb_str_subseq,
-    .str_new_frozen = rb_str_new_frozen,
-    .str_buf_new = rb_str_buf_new,
-    .str_buf_cat = rb_str_buf_cat,
-    .str_modify = rb_str_modify,
-    .str_set_len = rb_str_set_len,
-    .str_cat = rb_str_cat,
     .str_resize = rb_str_resize,
     .str_new = rb_str_new,
     .str_new_cstr = rb_str_new_cstr,
-    .fstring = rb_fstring,
-    .is_ascii_string = is_ascii_string2,
+    .str_to_interned_str = rb_str_to_interned_str,
     .enc_str_new = enc_str_new,
-    .enc_str_buf_cat = enc_str_buf_cat,
-    .str_buf_append = rb_str_buf_append,
     .str_vcatf = rb_str_vcatf,
-    .string_value_cstr = rb_string_value_cstr,
     .rb_sprintf = rb_sprintf,
     .rstring_ptr = RSTRING_PTR,
-    .rstring_end = RSTRING_END,
     .rstring_len = RSTRING_LEN,
-    .filesystem_str_new_cstr = rb_filesystem_str_new_cstr,
-    .obj_as_string = rb_obj_as_string,
 
-    .hash_clear = rb_hash_clear,
-    .hash_new = rb_hash_new,
-    .hash_aset = rb_hash_aset,
-    .hash_delete = rb_hash_delete,
-    .hash_lookup = rb_hash_lookup,
-    .ident_hash_new = rb_ident_hash_new,
-
-    .num2int = rb_num2int_inline,
     .int2num = rb_int2num_inline,
 
     .stderr_tty_p = rb_stderr_tty_p,
     .write_error_str = rb_write_error_str,
-    .default_rs = default_rs,
     .io_write = rb_io_write,
     .io_flush = rb_io_flush,
     .io_puts = rb_io_puts,
-    .io_gets_internal = rb_io_gets_internal,
 
     .debug_output_stdout = rb_ractor_stdout,
     .debug_output_stderr = rb_ractor_stderr,
@@ -597,51 +384,33 @@ static const rb_parser_config_t rb_global_parser_config = {
     .enc_get = enc_get,
     .enc_asciicompat = enc_asciicompat,
     .utf8_encoding = utf8_encoding,
-    .enc_associate = enc_associate,
     .ascii8bit_encoding = ascii8bit_encoding,
     .enc_codelen = enc_codelen,
     .enc_mbcput = enc_mbcput,
-    .char_to_option_kcode = rb_char_to_option_kcode,
-    .ascii8bit_encindex = rb_ascii8bit_encindex,
     .enc_find_index = rb_enc_find_index,
     .enc_from_index = enc_from_index,
-    .enc_associate_index = rb_enc_associate_index,
     .enc_isspace = enc_isspace,
     .enc_coderange_7bit = ENC_CODERANGE_7BIT,
     .enc_coderange_unknown = ENC_CODERANGE_UNKNOWN,
-    .enc_compatible = enc_compatible,
-    .enc_from_encoding = enc_from_encoding,
-    .encoding_get = encoding_get,
-    .encoding_set = encoding_set,
-    .encoding_is_ascii8bit = encoding_is_ascii8bit,
-    .usascii_encoding = usascii_encoding,
-
-    .ractor_make_shareable = rb_ractor_make_shareable,
+    .enc_mbminlen = enc_mbminlen,
+    .enc_isascii = enc_isascii,
+    .enc_mbc_to_codepoint = enc_mbc_to_codepoint,
 
     .local_defined = local_defined,
     .dvar_defined = dvar_defined,
 
-    .literal_cmp = literal_cmp,
-    .literal_hash = literal_hash,
-
     .syntax_error_append = syntax_error_append,
     .raise = rb_raise,
     .syntax_error_new = syntax_error_new,
 
     .errinfo = rb_errinfo,
     .set_errinfo = rb_set_errinfo,
-    .exc_raise = rb_exc_raise,
     .make_exception = rb_make_exception,
 
     .sized_xfree = ruby_sized_xfree,
     .sized_realloc_n = ruby_sized_realloc_n,
-    .obj_write = obj_write,
-    .obj_written = obj_written,
-    .gc_register_mark_object = rb_gc_register_mark_object,
     .gc_guard = gc_guard,
     .gc_mark = rb_gc_mark,
-    .gc_mark_movable = rb_gc_mark_movable,
-    .gc_location = rb_gc_location,
 
     .reg_compile = rb_reg_compile,
     .reg_check_preprocess = rb_reg_check_preprocess,
@@ -661,31 +430,92 @@ static const rb_parser_config_t rb_global_parser_config = {
     .scan_digits = ruby_scan_digits,
     .strtod = ruby_strtod,
 
-    .rbool = rbool,
-    .undef_p = undef_p,
     .rtest = rtest,
     .nil_p = nil_p,
     .qnil = Qnil,
-    .qtrue = Qtrue,
     .qfalse = Qfalse,
-    .qundef = Qundef,
     .eArgError = arg_error,
-    .mRubyVMFrozenCore = ruby_vm_frozen_core,
     .long2int = rb_long2int,
-    .special_const_p = special_const_p,
-    .builtin_type = builtin_type,
-
-    .node_case_when_optimizable_literal = rb_node_case_when_optimizable_literal,
 
     /* For Ripper */
     .static_id2sym = static_id2sym,
     .str_coderange_scan_restartable = str_coderange_scan_restartable,
 };
+#endif
 
-rb_parser_t *
-rb_parser_params_allocate(void)
+enum lex_type {
+    lex_type_str,
+    lex_type_io,
+    lex_type_array,
+    lex_type_generic,
+};
+
+struct ruby_parser {
+    rb_parser_t *parser_params;
+    enum lex_type type;
+    union {
+        struct lex_pointer_string lex_str;
+        struct {
+            VALUE file;
+        } lex_io;
+        struct {
+            VALUE ary;
+        } lex_array;
+    } data;
+};
+
+static void
+parser_mark(void *ptr)
+{
+    struct ruby_parser *parser = (struct ruby_parser*)ptr;
+    rb_ruby_parser_mark(parser->parser_params);
+
+    switch (parser->type) {
+      case lex_type_str:
+        rb_gc_mark(parser->data.lex_str.str);
+        break;
+      case lex_type_io:
+        rb_gc_mark(parser->data.lex_io.file);
+        break;
+      case lex_type_array:
+        rb_gc_mark(parser->data.lex_array.ary);
+        break;
+      case lex_type_generic:
+        /* noop. Caller of rb_parser_compile_generic should mark the objects. */
+        break;
+    }
+}
+
+static void
+parser_free(void *ptr)
+{
+    struct ruby_parser *parser = (struct ruby_parser*)ptr;
+    rb_ruby_parser_free(parser->parser_params);
+    xfree(parser);
+}
+
+static size_t
+parser_memsize(const void *ptr)
+{
+    struct ruby_parser *parser = (struct ruby_parser*)ptr;
+    return rb_ruby_parser_memsize(parser->parser_params);
+}
+
+static const rb_data_type_t ruby_parser_data_type = {
+    "parser",
+    {
+        parser_mark,
+        parser_free,
+        parser_memsize,
+    },
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+#ifdef UNIVERSAL_PARSER
+const rb_parser_config_t *
+rb_ruby_parser_config(void)
 {
-    return rb_ruby_parser_allocate(&rb_global_parser_config);
+    return &rb_global_parser_config;
 }
 
 rb_parser_t *
@@ -693,6 +523,13 @@ rb_parser_params_new(void)
 {
     return rb_ruby_parser_new(&rb_global_parser_config);
 }
+#else
+rb_parser_t *
+rb_parser_params_new(void)
+{
+    return rb_ruby_parser_new();
+}
+#endif /* UNIVERSAL_PARSER */
 
 VALUE
 rb_parser_new(void)
@@ -733,12 +570,12 @@ rb_parser_set_context(VALUE vparser, const struct rb_iseq_struct *base, int main
 }
 
 void
-rb_parser_set_script_lines(VALUE vparser, VALUE lines)
+rb_parser_set_script_lines(VALUE vparser)
 {
     struct ruby_parser *parser;
 
     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
-    rb_ruby_parser_set_script_lines(parser->parser_params, lines);
+    rb_ruby_parser_set_script_lines(parser->parser_params);
 }
 
 void
@@ -750,65 +587,227 @@ rb_parser_error_tolerant(VALUE vparser)
     rb_ruby_parser_error_tolerant(parser->parser_params);
 }
 
-rb_ast_t*
+void
+rb_parser_keep_tokens(VALUE vparser)
+{
+    struct ruby_parser *parser;
+
+    TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
+    rb_ruby_parser_keep_tokens(parser->parser_params);
+}
+
+rb_parser_string_t *
+rb_parser_lex_get_str(struct parser_params *p, struct lex_pointer_string *ptr_str)
+{
+    char *beg, *end, *start;
+    long len;
+    VALUE s = ptr_str->str;
+
+    beg = RSTRING_PTR(s);
+    len = RSTRING_LEN(s);
+    start = beg;
+    if (ptr_str->ptr) {
+        if (len == ptr_str->ptr) return 0;
+        beg += ptr_str->ptr;
+        len -= ptr_str->ptr;
+    }
+    end = memchr(beg, '\n', len);
+    if (end) len = ++end - beg;
+    ptr_str->ptr += len;
+    return rb_str_to_parser_string(p, rb_str_subseq(s, beg - start, len));
+}
+
+static rb_parser_string_t *
+lex_get_str(struct parser_params *p, rb_parser_input_data input, int line_count)
+{
+    return rb_parser_lex_get_str(p, (struct lex_pointer_string *)input);
+}
+
+static void parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *lines);
+
+static rb_ast_t*
+parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, VALUE fname, rb_parser_input_data input, int line)
+{
+    rb_ast_t *ast = rb_parser_compile(p, gets, fname, input, line);
+    parser_aset_script_lines_for(fname, ast->body.script_lines);
+    return ast;
+}
+
+static rb_ast_t*
+parser_compile_string0(struct ruby_parser *parser, VALUE fname, VALUE s, int line)
+{
+    VALUE str = rb_str_new_frozen(s);
+
+    parser->type = lex_type_str;
+    parser->data.lex_str.str = str;
+    parser->data.lex_str.ptr = 0;
+
+    return parser_compile(parser->parser_params, lex_get_str, fname, (rb_parser_input_data)&parser->data, line);
+}
+
+static rb_encoding *
+must_be_ascii_compatible(VALUE s)
+{
+    rb_encoding *enc = rb_enc_get(s);
+    if (!rb_enc_asciicompat(enc)) {
+        rb_raise(rb_eArgError, "invalid source encoding");
+    }
+    return enc;
+}
+
+static rb_ast_t*
+parser_compile_string_path(struct ruby_parser *parser, VALUE f, VALUE s, int line)
+{
+    must_be_ascii_compatible(s);
+    return parser_compile_string0(parser, f, s, line);
+}
+
+static rb_ast_t*
+parser_compile_string(struct ruby_parser *parser, const char *f, VALUE s, int line)
+{
+    return parser_compile_string_path(parser, rb_filesystem_str_new_cstr(f), s, line);
+}
+
+VALUE rb_io_gets_internal(VALUE io);
+
+static rb_parser_string_t *
+lex_io_gets(struct parser_params *p, rb_parser_input_data input, int line_count)
+{
+    VALUE io = (VALUE)input;
+    VALUE line = rb_io_gets_internal(io);
+    if (NIL_P(line)) return 0;
+    return rb_str_to_parser_string(p, line);
+}
+
+static rb_parser_string_t *
+lex_gets_array(struct parser_params *p, rb_parser_input_data data, int index)
+{
+    VALUE array = (VALUE)data;
+    VALUE str = rb_ary_entry(array, index);
+    if (!NIL_P(str)) {
+        StringValue(str);
+        if (!rb_enc_asciicompat(rb_enc_get(str))) {
+            rb_raise(rb_eArgError, "invalid source encoding");
+        }
+        return rb_str_to_parser_string(p, str);
+    }
+    else {
+        return 0;
+    }
+}
+
+static rb_ast_t*
+parser_compile_file_path(struct ruby_parser *parser, VALUE fname, VALUE file, int start)
+{
+    parser->type = lex_type_io;
+    parser->data.lex_io.file = file;
+
+    return parser_compile(parser->parser_params, lex_io_gets, fname, (rb_parser_input_data)file, start);
+}
+
+static rb_ast_t*
+parser_compile_array(struct ruby_parser *parser, VALUE fname, VALUE array, int start)
+{
+    parser->type = lex_type_array;
+    parser->data.lex_array.ary = array;
+
+    return parser_compile(parser->parser_params, lex_gets_array, fname, (rb_parser_input_data)array, start);
+}
+
+static rb_ast_t*
+parser_compile_generic(struct ruby_parser *parser, rb_parser_lex_gets_func *lex_gets, VALUE fname, VALUE input, int start)
+{
+    parser->type = lex_type_generic;
+
+    return parser_compile(parser->parser_params, lex_gets, fname, (rb_parser_input_data)input, start);
+}
+
+static void
+ast_free(void *ptr)
+{
+    rb_ast_t *ast = (rb_ast_t *)ptr;
+    rb_ast_free(ast);
+}
+
+static const rb_data_type_t ast_data_type = {
+    "AST",
+    {
+        NULL,
+        ast_free,
+        NULL, // No dsize() because this object does not appear in ObjectSpace.
+    },
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE
+ast_alloc(void)
+{
+    return TypedData_Wrap_Struct(0, &ast_data_type, NULL);
+}
+
+VALUE
 rb_parser_compile_file_path(VALUE vparser, VALUE fname, VALUE file, int start)
 {
     struct ruby_parser *parser;
-    rb_ast_t *ast;
+    VALUE ast_value = ast_alloc();
 
     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
-    ast = rb_ruby_parser_compile_file_path(parser->parser_params, fname, file, start);
+    DATA_PTR(ast_value) = parser_compile_file_path(parser, fname, file, start);
     RB_GC_GUARD(vparser);
 
-    return ast;
+    return ast_value;
 }
 
-void
-rb_parser_keep_tokens(VALUE vparser)
+VALUE
+rb_parser_compile_array(VALUE vparser, VALUE fname, VALUE array, int start)
 {
     struct ruby_parser *parser;
+    VALUE ast_value = ast_alloc();
 
     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
-    rb_ruby_parser_keep_tokens(parser->parser_params);
+    DATA_PTR(ast_value) = parser_compile_array(parser, fname, array, start);
+    RB_GC_GUARD(vparser);
+
+    return ast_value;
 }
 
-rb_ast_t*
-rb_parser_compile_generic(VALUE vparser, VALUE (*lex_gets)(VALUE, int), VALUE fname, VALUE input, int start)
+VALUE
+rb_parser_compile_generic(VALUE vparser, rb_parser_lex_gets_func *lex_gets, VALUE fname, VALUE input, int start)
 {
     struct ruby_parser *parser;
-    rb_ast_t *ast;
+    VALUE ast_value = ast_alloc();
 
     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
-    ast = rb_ruby_parser_compile_generic(parser->parser_params, lex_gets, fname, input, start);
+    DATA_PTR(ast_value) = parser_compile_generic(parser, lex_gets, fname, input, start);
     RB_GC_GUARD(vparser);
 
-    return ast;
+    return ast_value;
 }
 
-rb_ast_t*
+VALUE
 rb_parser_compile_string(VALUE vparser, const char *f, VALUE s, int line)
 {
     struct ruby_parser *parser;
-    rb_ast_t *ast;
+    VALUE ast_value = ast_alloc();
 
     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
-    ast = rb_ruby_parser_compile_string(parser->parser_params, f, s, line);
+    DATA_PTR(ast_value) = parser_compile_string(parser, f, s, line);
     RB_GC_GUARD(vparser);
 
-    return ast;
+    return ast_value;
 }
 
-rb_ast_t*
+VALUE
 rb_parser_compile_string_path(VALUE vparser, VALUE f, VALUE s, int line)
 {
     struct ruby_parser *parser;
-    rb_ast_t *ast;
+    VALUE ast_value = ast_alloc();
 
     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
-    ast = rb_ruby_parser_compile_string_path(parser->parser_params, f, s, line);
+    DATA_PTR(ast_value) = parser_compile_string_path(parser, f, s, line);
     RB_GC_GUARD(vparser);
 
-    return ast;
+    return ast_value;
 }
 
 VALUE
@@ -817,7 +816,7 @@ rb_parser_encoding(VALUE vparser)
     struct ruby_parser *parser;
 
     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
-    return rb_ruby_parser_encoding(parser->parser_params);
+    return rb_enc_from_encoding(rb_ruby_parser_encoding(parser->parser_params));
 }
 
 VALUE
@@ -838,11 +837,50 @@ rb_parser_set_yydebug(VALUE vparser, VALUE flag)
     rb_ruby_parser_set_yydebug(parser->parser_params, RTEST(flag));
     return flag;
 }
-#endif
+
+void
+rb_set_script_lines_for(VALUE vparser, VALUE path)
+{
+    struct ruby_parser *parser;
+    VALUE hash;
+    ID script_lines;
+    CONST_ID(script_lines, "SCRIPT_LINES__");
+    if (!rb_const_defined_at(rb_cObject, script_lines)) return;
+    hash = rb_const_get_at(rb_cObject, script_lines);
+    if (RB_TYPE_P(hash, T_HASH)) {
+        rb_hash_aset(hash, path, Qtrue);
+        TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
+        rb_ruby_parser_set_script_lines(parser->parser_params);
+    }
+}
+
+VALUE
+rb_parser_build_script_lines_from(rb_parser_ary_t *lines)
+{
+    int i;
+    if (!lines) return Qnil;
+    if (lines->data_type != PARSER_ARY_DATA_SCRIPT_LINE) {
+        rb_bug("unexpected rb_parser_ary_data_type (%d) for script lines", lines->data_type);
+    }
+    VALUE script_lines = rb_ary_new_capa(lines->len);
+    for (i = 0; i < lines->len; i++) {
+        rb_parser_string_t *str = (rb_parser_string_t *)lines->data[i];
+        rb_ary_push(script_lines, rb_enc_str_new(str->ptr, str->len, str->enc));
+    }
+    return script_lines;
+}
 
 VALUE
 rb_str_new_parser_string(rb_parser_string_t *str)
 {
+    VALUE string = rb_enc_literal_str(str->ptr, str->len, str->enc);
+    rb_enc_str_coderange(string);
+    return string;
+}
+
+VALUE
+rb_str_new_mutable_parser_string(rb_parser_string_t *str)
+{
     return rb_enc_str_new(str->ptr, str->len, str->enc);
 }
 
@@ -1011,6 +1049,16 @@ rb_node_dregx_string_val(const NODE *node)
 }
 
 VALUE
+rb_node_regx_string_val(const NODE *node)
+{
+    rb_node_regx_t *node_reg = RNODE_REGX(node);
+    rb_parser_string_t *string = node_reg->string;
+    VALUE str = rb_enc_str_new(string->ptr, string->len, string->enc);
+
+    return rb_reg_compile(str, node_reg->options, NULL, 0);
+}
+
+VALUE
 rb_node_line_lineno_val(const NODE *node)
 {
     return INT2FIX(node->nd_loc.beg_pos.lineno);
@@ -1028,53 +1076,44 @@ rb_node_encoding_val(const NODE *node)
     return rb_enc_from_encoding(RNODE_ENCODING(node)->enc);
 }
 
+static void
+parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *lines)
+{
+    VALUE hash, script_lines;
+    ID script_lines_id;
+    if (NIL_P(path) || !lines) return;
+    CONST_ID(script_lines_id, "SCRIPT_LINES__");
+    if (!rb_const_defined_at(rb_cObject, script_lines_id)) return;
+    hash = rb_const_get_at(rb_cObject, script_lines_id);
+    if (!RB_TYPE_P(hash, T_HASH)) return;
+    if (rb_hash_lookup(hash, path) == Qnil) return;
+    script_lines = rb_parser_build_script_lines_from(lines);
+    rb_hash_aset(hash, path, script_lines);
+}
+
 VALUE
-rb_node_const_decl_val(const NODE *node)
-{
-    VALUE path;
-    switch (nd_type(node)) {
-      case NODE_CDECL:
-        if (RNODE_CDECL(node)->nd_vid) {
-            path = rb_id2str(RNODE_CDECL(node)->nd_vid);
-            goto end;
-        }
-        else {
-            node = RNODE_CDECL(node)->nd_else;
-        }
-        break;
-      case NODE_COLON2:
-        break;
-      case NODE_COLON3:
-        // ::Const
-        path = rb_str_new_cstr("::");
-        rb_str_append(path, rb_id2str(RNODE_COLON3(node)->nd_mid));
-        goto end;
-      default:
-        rb_bug("unexpected node: %s", ruby_node_name(nd_type(node)));
-        UNREACHABLE_RETURN(0);
-    }
+rb_ruby_ast_new(const NODE *const root)
+{
+    rb_ast_t *ast;
+    VALUE ast_value = TypedData_Make_Struct(0, rb_ast_t, &ast_data_type, ast);
+#ifdef UNIVERSAL_PARSER
+    ast->config = &rb_global_parser_config;
+#endif
+    ast->body = (rb_ast_body_t){
+        .root = root,
+        .frozen_string_literal = -1,
+        .coverage_enabled = -1,
+        .script_lines = NULL,
+        .line_count = 0,
+    };
+    return ast_value;
+}
 
-    path = rb_ary_new();
-    if (node) {
-        for (; node && nd_type_p(node, NODE_COLON2); node = RNODE_COLON2(node)->nd_head) {
-            rb_ary_push(path, rb_id2str(RNODE_COLON2(node)->nd_mid));
-        }
-        if (node && nd_type_p(node, NODE_CONST)) {
-            // Const::Name
-            rb_ary_push(path, rb_id2str(RNODE_CONST(node)->nd_vid));
-        }
-        else if (node && nd_type_p(node, NODE_COLON3)) {
-            // ::Const::Name
-            rb_ary_push(path, rb_id2str(RNODE_COLON3(node)->nd_mid));
-            rb_ary_push(path, rb_str_new(0, 0));
-        }
-        else {
-            // expression::Name
-            rb_ary_push(path, rb_str_new_cstr("..."));
-        }
-        path = rb_ary_join(rb_ary_reverse(path), rb_str_new_cstr("::"));
-    }
-  end:
-    path = rb_fstring(path);
-    return path;
+rb_ast_t *
+rb_ruby_ast_data_get(VALUE ast_value)
+{
+    rb_ast_t *ast;
+    if (NIL_P(ast_value)) return NULL;
+    TypedData_Get_Struct(ast_value, rb_ast_t, &ast_data_type, ast);
+    return ast;
 }