ruby_parser.c

   1 /* This is a wrapper for parse.y */
   2
   3 #include "internal/parse.h"
   4 #include "internal/re.h"
   5 #include "internal/ruby_parser.h"
   6
   7 #include "node.h"
   8 #include "rubyparser.h"
   9 #include "internal/error.h"
  10
  11 #ifdef UNIVERSAL_PARSER
  12
  13 #include "internal.h"
  14 #include "internal/array.h"
  15 #include "internal/bignum.h"
  16 #include "internal/compile.h"
  17 #include "internal/complex.h"
  18 #include "internal/encoding.h"
  19 #include "internal/gc.h"
  20 #include "internal/hash.h"
  21 #include "internal/io.h"
  22 #include "internal/rational.h"
  23 #include "internal/re.h"
  24 #include "internal/string.h"
  25 #include "internal/symbol.h"
  26 #include "internal/thread.h"
  27
  28 #include "ruby/ractor.h"
  29 #include "ruby/ruby.h"
  30 #include "ruby/util.h"
  31 #include "internal.h"
  32 #include "vm_core.h"
  33 #include "symbol.h"
  34
  35 #define parser_encoding const void
  36
  37 RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 6, 0)
  38 static VALUE
  39 syntax_error_append(VALUE exc, VALUE file, int line, int column,
  40                     parser_encoding *enc, const char *fmt, va_list args)
  41 {
  42     return rb_syntax_error_append(exc, file, line, column, enc, fmt, args);
  43 }
  44
  45 static int
  46 local_defined(ID id, const void *p)
  47 {
  48     return rb_local_defined(id, (const rb_iseq_t *)p);
  49 }
  50
  51 static int
  52 dvar_defined(ID id, const void *p)
  53 {
  54     return rb_dvar_defined(id, (const rb_iseq_t *)p);
  55 }
  56
  57 static int
  58 is_usascii_enc(parser_encoding *enc)
  59 {
  60     return rb_is_usascii_enc(enc);
  61 }
  62
  63 static int
  64 is_local_id2(ID id)
  65 {
  66     return is_local_id(id);
  67 }
  68
  69 static int
  70 is_attrset_id2(ID id)
  71 {
  72     return is_attrset_id(id);
  73 }
  74
  75 static int
  76 is_notop_id2(ID id)
  77 {
  78     return is_notop_id(id);
  79 }
  80
  81 static VALUE
  82 enc_str_new(const char *ptr, long len, parser_encoding *enc)
  83 {
  84     return rb_enc_str_new(ptr, len, enc);
  85 }
  86
  87 static int
  88 enc_isalnum(OnigCodePoint c, parser_encoding *enc)
  89 {
  90     return rb_enc_isalnum(c, enc);
  91 }
  92
  93 static int
  94 enc_precise_mbclen(const char *p, const char *e, parser_encoding *enc)
  95 {
  96     return rb_enc_precise_mbclen(p, e, enc);
  97 }
  98
  99 static int
 100 mbclen_charfound_p(int len)
 101 {
 102     return MBCLEN_CHARFOUND_P(len);
 103 }
 104
 105 static int
 106 mbclen_charfound_len(int len)
 107 {
 108     return MBCLEN_CHARFOUND_LEN(len);
 109 }
 110
 111 static const char *
 112 enc_name(parser_encoding *enc)
 113 {
 114     return rb_enc_name(enc);
 115 }
 116
 117 static char *
 118 enc_prev_char(const char *s, const char *p, const char *e, parser_encoding *enc)
 119 {
 120     return rb_enc_prev_char(s, p, e, enc);
 121 }
 122
 123 static parser_encoding *
 124 enc_get(VALUE obj)
 125 {
 126     return rb_enc_get(obj);
 127 }
 128
 129 static int
 130 enc_asciicompat(parser_encoding *enc)
 131 {
 132     return rb_enc_asciicompat(enc);
 133 }
 134
 135 static parser_encoding *
 136 utf8_encoding(void)
 137 {
 138     return rb_utf8_encoding();
 139 }
 140
 141 static VALUE
 142 enc_associate(VALUE obj, parser_encoding *enc)
 143 {
 144     return rb_enc_associate(obj, enc);
 145 }
 146
 147 static parser_encoding *
 148 ascii8bit_encoding(void)
 149 {
 150     return rb_ascii8bit_encoding();
 151 }
 152
 153 static int
 154 enc_codelen(int c, parser_encoding *enc)
 155 {
 156     return rb_enc_codelen(c, enc);
 157 }
 158
 159 static int
 160 enc_mbcput(unsigned int c, void *buf, parser_encoding *enc)
 161 {
 162     return rb_enc_mbcput(c, buf, enc);
 163 }
 164
 165 static parser_encoding *
 166 enc_from_index(int idx)
 167 {
 168     return rb_enc_from_index(idx);
 169 }
 170
 171 static int
 172 enc_isspace(OnigCodePoint c, parser_encoding *enc)
 173 {
 174     return rb_enc_isspace(c, enc);
 175 }
 176
 177 static ID
 178 intern3(const char *name, long len, parser_encoding *enc)
 179 {
 180     return rb_intern3(name, len, enc);
 181 }
 182
 183 static parser_encoding *
 184 usascii_encoding(void)
 185 {
 186     return rb_usascii_encoding();
 187 }
 188
 189 static int
 190 enc_symname_type(const char *name, long len, parser_encoding *enc, unsigned int allowed_attrset)
 191 {
 192     return rb_enc_symname_type(name, len, enc, allowed_attrset);
 193 }
 194
 195 typedef struct {
 196     struct parser_params *parser;
 197     rb_encoding *enc;
 198     NODE *succ_block;
 199     const rb_code_location_t *loc;
 200     rb_parser_assignable_func assignable;
 201 } reg_named_capture_assign_t;
 202
 203 static int
 204 reg_named_capture_assign_iter(const OnigUChar *name, const OnigUChar *name_end,
 205           int back_num, int *back_refs, OnigRegex regex, void *arg0)
 206 {
 207     reg_named_capture_assign_t *arg = (reg_named_capture_assign_t*)arg0;
 208     struct parser_params* p = arg->parser;
 209     rb_encoding *enc = arg->enc;
 210     const rb_code_location_t *loc = arg->loc;
 211     long len = name_end - name;
 212     const char *s = (const char *)name;
 213
 214     return rb_reg_named_capture_assign_iter_impl(p, s, len, enc, &arg->succ_block, loc, arg->assignable);
 215 }
 216
 217 static NODE *
 218 reg_named_capture_assign(struct parser_params* p, VALUE regexp, const rb_code_location_t *loc,
 219                          rb_parser_assignable_func assignable)
 220 {
 221     reg_named_capture_assign_t arg;
 222
 223     arg.parser = p;
 224     arg.enc = rb_enc_get(regexp);
 225     arg.succ_block = 0;
 226     arg.loc = loc;
 227     arg.assignable = assignable;
 228     onig_foreach_name(RREGEXP_PTR(regexp), reg_named_capture_assign_iter, &arg);
 229
 230     if (!arg.succ_block) return 0;
 231     return RNODE_BLOCK(arg.succ_block)->nd_next;
 232 }
 233
 234 static int
 235 rtest(VALUE obj)
 236 {
 237     return (int)RB_TEST(obj);
 238 }
 239
 240 static int
 241 nil_p(VALUE obj)
 242 {
 243     return (int)NIL_P(obj);
 244 }
 245
 246 static VALUE
 247 syntax_error_new(void)
 248 {
 249     return rb_class_new_instance(0, 0, rb_eSyntaxError);
 250 }
 251
 252 static void *
 253 memmove2(void *dest, const void *src, size_t t, size_t n)
 254 {
 255     return memmove(dest, src, rbimpl_size_mul_or_raise(t, n));
 256 }
 257
 258 static void *
 259 nonempty_memcpy(void *dest, const void *src, size_t t, size_t n)
 260 {
 261     return ruby_nonempty_memcpy(dest, src, rbimpl_size_mul_or_raise(t, n));
 262 }
 263
 264 static VALUE
 265 ruby_verbose2(void)
 266 {
 267     return ruby_verbose;
 268 }
 269
 270 static int *
 271 rb_errno_ptr2(void)
 272 {
 273     return rb_errno_ptr();
 274 }
 275
 276 static void *
 277 zalloc(size_t elemsiz)
 278 {
 279     return ruby_xcalloc(1, elemsiz);
 280 }
 281
 282 static void
 283 gc_guard(VALUE obj)
 284 {
 285     RB_GC_GUARD(obj);
 286 }
 287
 288 static VALUE
 289 arg_error(void)
 290 {
 291     return rb_eArgError;
 292 }
 293
 294 static VALUE
 295 static_id2sym(ID id)
 296 {
 297     return (((VALUE)(id)<<RUBY_SPECIAL_SHIFT)|SYMBOL_FLAG);
 298 }
 299
 300 static long
 301 str_coderange_scan_restartable(const char *s, const char *e, parser_encoding *enc, int *cr)
 302 {
 303     return rb_str_coderange_scan_restartable(s, e, enc, cr);
 304 }
 305
 306 static int
 307 enc_mbminlen(parser_encoding *enc)
 308 {
 309     return rb_enc_mbminlen(enc);
 310 }
 311
 312 static bool
 313 enc_isascii(OnigCodePoint c, parser_encoding *enc)
 314 {
 315     return rb_enc_isascii(c, enc);
 316 }
 317
 318 static OnigCodePoint
 319 enc_mbc_to_codepoint(const char *p, const char *e, parser_encoding *enc)
 320 {
 321     const OnigUChar *up = RBIMPL_CAST((const OnigUChar *)p);
 322     const OnigUChar *ue = RBIMPL_CAST((const OnigUChar *)e);
 323
 324     return ONIGENC_MBC_TO_CODE((rb_encoding *)enc, up, ue);
 325 }
 326
 327 extern VALUE rb_eArgError;
 328
 329 static const rb_parser_config_t rb_global_parser_config = {
 330     .malloc = ruby_xmalloc,
 331     .calloc = ruby_xcalloc,
 332     .realloc = ruby_xrealloc,
 333     .free = ruby_xfree,
 334     .alloc_n = ruby_xmalloc2,
 335     .alloc = ruby_xmalloc,
 336     .realloc_n = ruby_xrealloc2,
 337     .zalloc = zalloc,
 338     .rb_memmove = memmove2,
 339     .nonempty_memcpy = nonempty_memcpy,
 340     .xmalloc_mul_add = rb_xmalloc_mul_add,
 341
 342     .compile_callback = rb_suppress_tracing,
 343     .reg_named_capture_assign = reg_named_capture_assign,
 344
 345     .attr_get = rb_attr_get,
 346
 347     .ary_new_from_args = rb_ary_new_from_args,
 348     .ary_unshift = rb_ary_unshift,
 349
 350     .make_temporary_id = rb_make_temporary_id,
 351     .is_local_id = is_local_id2,
 352     .is_attrset_id = is_attrset_id2,
 353     .is_global_name_punct = is_global_name_punct,
 354     .id_type = id_type,
 355     .id_attrset = rb_id_attrset,
 356     .intern = rb_intern,
 357     .intern2 = rb_intern2,
 358     .intern3 = intern3,
 359     .intern_str = rb_intern_str,
 360     .is_notop_id = is_notop_id2,
 361     .enc_symname_type = enc_symname_type,
 362     .id2name = rb_id2name,
 363     .id2str = rb_id2str,
 364     .id2sym = rb_id2sym,
 365     .sym2id = rb_sym2id,
 366
 367     .str_catf = rb_str_catf,
 368     .str_cat_cstr = rb_str_cat_cstr,
 369     .str_resize = rb_str_resize,
 370     .str_new = rb_str_new,
 371     .str_new_cstr = rb_str_new_cstr,
 372     .str_to_interned_str = rb_str_to_interned_str,
 373     .enc_str_new = enc_str_new,
 374     .str_vcatf = rb_str_vcatf,
 375     .rb_sprintf = rb_sprintf,
 376     .rstring_ptr = RSTRING_PTR,
 377     .rstring_len = RSTRING_LEN,
 378     .obj_as_string = rb_obj_as_string,
 379
 380     .int2num = rb_int2num_inline,
 381
 382     .stderr_tty_p = rb_stderr_tty_p,
 383     .write_error_str = rb_write_error_str,
 384     .io_write = rb_io_write,
 385     .io_flush = rb_io_flush,
 386     .io_puts = rb_io_puts,
 387
 388     .debug_output_stdout = rb_ractor_stdout,
 389     .debug_output_stderr = rb_ractor_stderr,
 390
 391     .is_usascii_enc = is_usascii_enc,
 392     .enc_isalnum = enc_isalnum,
 393     .enc_precise_mbclen = enc_precise_mbclen,
 394     .mbclen_charfound_p = mbclen_charfound_p,
 395     .mbclen_charfound_len = mbclen_charfound_len,
 396     .enc_name = enc_name,
 397     .enc_prev_char = enc_prev_char,
 398     .enc_get = enc_get,
 399     .enc_asciicompat = enc_asciicompat,
 400     .utf8_encoding = utf8_encoding,
 401     .enc_associate = enc_associate,
 402     .ascii8bit_encoding = ascii8bit_encoding,
 403     .enc_codelen = enc_codelen,
 404     .enc_mbcput = enc_mbcput,
 405     .enc_find_index = rb_enc_find_index,
 406     .enc_from_index = enc_from_index,
 407     .enc_isspace = enc_isspace,
 408     .enc_coderange_7bit = ENC_CODERANGE_7BIT,
 409     .enc_coderange_unknown = ENC_CODERANGE_UNKNOWN,
 410     .usascii_encoding = usascii_encoding,
 411     .enc_mbminlen = enc_mbminlen,
 412     .enc_isascii = enc_isascii,
 413     .enc_mbc_to_codepoint = enc_mbc_to_codepoint,
 414
 415     .local_defined = local_defined,
 416     .dvar_defined = dvar_defined,
 417
 418     .syntax_error_append = syntax_error_append,
 419     .raise = rb_raise,
 420     .syntax_error_new = syntax_error_new,
 421
 422     .errinfo = rb_errinfo,
 423     .set_errinfo = rb_set_errinfo,
 424     .exc_raise = rb_exc_raise,
 425     .make_exception = rb_make_exception,
 426
 427     .sized_xfree = ruby_sized_xfree,
 428     .sized_realloc_n = ruby_sized_realloc_n,
 429     .gc_guard = gc_guard,
 430     .gc_mark = rb_gc_mark,
 431
 432     .reg_compile = rb_reg_compile,
 433     .reg_check_preprocess = rb_reg_check_preprocess,
 434     .memcicmp = rb_memcicmp,
 435
 436     .compile_warn = rb_compile_warn,
 437     .compile_warning = rb_compile_warning,
 438     .bug = rb_bug,
 439     .fatal = rb_fatal,
 440     .verbose = ruby_verbose2,
 441     .errno_ptr = rb_errno_ptr2,
 442
 443     .make_backtrace = rb_make_backtrace,
 444
 445     .scan_hex = ruby_scan_hex,
 446     .scan_oct = ruby_scan_oct,
 447     .scan_digits = ruby_scan_digits,
 448     .strtod = ruby_strtod,
 449
 450     .rtest = rtest,
 451     .nil_p = nil_p,
 452     .qnil = Qnil,
 453     .qfalse = Qfalse,
 454     .eArgError = arg_error,
 455     .long2int = rb_long2int,
 456
 457     /* For Ripper */
 458     .static_id2sym = static_id2sym,
 459     .str_coderange_scan_restartable = str_coderange_scan_restartable,
 460 };
 461 #endif
 462
 463 enum lex_type {
 464     lex_type_str,
 465     lex_type_io,
 466     lex_type_array,
 467     lex_type_generic,
 468 };
 469
 470 struct ruby_parser {
 471     rb_parser_t *parser_params;
 472     enum lex_type type;
 473     union {
 474         struct lex_pointer_string lex_str;
 475         struct {
 476             VALUE file;
 477         } lex_io;
 478         struct {
 479             VALUE ary;
 480         } lex_array;
 481     } data;
 482 };
 483
 484 static void
 485 parser_mark(void *ptr)
 486 {
 487     struct ruby_parser *parser = (struct ruby_parser*)ptr;
 488     rb_ruby_parser_mark(parser->parser_params);
 489
 490     switch (parser->type) {
 491       case lex_type_str:
 492         rb_gc_mark(parser->data.lex_str.str);
 493         break;
 494       case lex_type_io:
 495         rb_gc_mark(parser->data.lex_io.file);
 496         break;
 497       case lex_type_array:
 498         rb_gc_mark(parser->data.lex_array.ary);
 499         break;
 500       case lex_type_generic:
 501         /* noop. Caller of rb_parser_compile_generic should mark the objects. */
 502         break;
 503     }
 504 }
 505
 506 static void
 507 parser_free(void *ptr)
 508 {
 509     struct ruby_parser *parser = (struct ruby_parser*)ptr;
 510     rb_ruby_parser_free(parser->parser_params);
 511     xfree(parser);
 512 }
 513
 514 static size_t
 515 parser_memsize(const void *ptr)
 516 {
 517     struct ruby_parser *parser = (struct ruby_parser*)ptr;
 518     return rb_ruby_parser_memsize(parser->parser_params);
 519 }
 520
 521 static const rb_data_type_t ruby_parser_data_type = {
 522     "parser",
 523     {
 524         parser_mark,
 525         parser_free,
 526         parser_memsize,
 527     },
 528     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
 529 };
 530
 531 #ifdef UNIVERSAL_PARSER
 532 const rb_parser_config_t *
 533 rb_ruby_parser_config(void)
 534 {
 535     return &rb_global_parser_config;
 536 }
 537
 538 rb_parser_t *
 539 rb_parser_params_new(void)
 540 {
 541     return rb_ruby_parser_new(&rb_global_parser_config);
 542 }
 543 #else
 544 rb_parser_t *
 545 rb_parser_params_new(void)
 546 {
 547     return rb_ruby_parser_new();
 548 }
 549 #endif /* UNIVERSAL_PARSER */
 550
 551 VALUE
 552 rb_parser_new(void)
 553 {
 554     struct ruby_parser *parser;
 555     rb_parser_t *parser_params;
 556
 557     /*
 558      * Create parser_params ahead of vparser because
 559      * rb_ruby_parser_new can run GC so if create vparser
 560      * first, parser_mark tries to mark not initialized parser_params.
 561      */
 562     parser_params = rb_parser_params_new();
 563     VALUE vparser = TypedData_Make_Struct(0, struct ruby_parser,
 564                                          &ruby_parser_data_type, parser);
 565     parser->parser_params = parser_params;
 566
 567     return vparser;
 568 }
 569
 570 void
 571 rb_parser_set_options(VALUE vparser, int print, int loop, int chomp, int split)
 572 {
 573     struct ruby_parser *parser;
 574
 575     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 576     rb_ruby_parser_set_options(parser->parser_params, print, loop, chomp, split);
 577 }
 578
 579 VALUE
 580 rb_parser_set_context(VALUE vparser, const struct rb_iseq_struct *base, int main)
 581 {
 582     struct ruby_parser *parser;
 583
 584     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 585     rb_ruby_parser_set_context(parser->parser_params, base, main);
 586     return vparser;
 587 }
 588
 589 void
 590 rb_parser_set_script_lines(VALUE vparser)
 591 {
 592     struct ruby_parser *parser;
 593
 594     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 595     rb_ruby_parser_set_script_lines(parser->parser_params);
 596 }
 597
 598 void
 599 rb_parser_error_tolerant(VALUE vparser)
 600 {
 601     struct ruby_parser *parser;
 602
 603     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 604     rb_ruby_parser_error_tolerant(parser->parser_params);
 605 }
 606
 607 void
 608 rb_parser_keep_tokens(VALUE vparser)
 609 {
 610     struct ruby_parser *parser;
 611
 612     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 613     rb_ruby_parser_keep_tokens(parser->parser_params);
 614 }
 615
 616 rb_parser_string_t *
 617 rb_parser_lex_get_str(struct parser_params *p, struct lex_pointer_string *ptr_str)
 618 {
 619     char *beg, *end, *start;
 620     long len;
 621     VALUE s = ptr_str->str;
 622
 623     beg = RSTRING_PTR(s);
 624     len = RSTRING_LEN(s);
 625     start = beg;
 626     if (ptr_str->ptr) {
 627         if (len == ptr_str->ptr) return 0;
 628         beg += ptr_str->ptr;
 629         len -= ptr_str->ptr;
 630     }
 631     end = memchr(beg, '\n', len);
 632     if (end) len = ++end - beg;
 633     ptr_str->ptr += len;
 634     return rb_str_to_parser_string(p, rb_str_subseq(s, beg - start, len));
 635 }
 636
 637 static rb_parser_string_t *
 638 lex_get_str(struct parser_params *p, rb_parser_input_data input, int line_count)
 639 {
 640     return rb_parser_lex_get_str(p, (struct lex_pointer_string *)input);
 641 }
 642
 643 static void parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *lines);
 644
 645 static rb_ast_t*
 646 parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, VALUE fname, rb_parser_input_data input, int line)
 647 {
 648     rb_ast_t *ast = rb_parser_compile(p, gets, fname, input, line);
 649     parser_aset_script_lines_for(fname, ast->body.script_lines);
 650     return ast;
 651 }
 652
 653 static rb_ast_t*
 654 parser_compile_string0(struct ruby_parser *parser, VALUE fname, VALUE s, int line)
 655 {
 656     VALUE str = rb_str_new_frozen(s);
 657
 658     parser->type = lex_type_str;
 659     parser->data.lex_str.str = str;
 660     parser->data.lex_str.ptr = 0;
 661
 662     return parser_compile(parser->parser_params, lex_get_str, fname, (rb_parser_input_data)&parser->data, line);
 663 }
 664
 665 static rb_encoding *
 666 must_be_ascii_compatible(VALUE s)
 667 {
 668     rb_encoding *enc = rb_enc_get(s);
 669     if (!rb_enc_asciicompat(enc)) {
 670         rb_raise(rb_eArgError, "invalid source encoding");
 671     }
 672     return enc;
 673 }
 674
 675 static rb_ast_t*
 676 parser_compile_string_path(struct ruby_parser *parser, VALUE f, VALUE s, int line)
 677 {
 678     must_be_ascii_compatible(s);
 679     return parser_compile_string0(parser, f, s, line);
 680 }
 681
 682 static rb_ast_t*
 683 parser_compile_string(struct ruby_parser *parser, const char *f, VALUE s, int line)
 684 {
 685     return parser_compile_string_path(parser, rb_filesystem_str_new_cstr(f), s, line);
 686 }
 687
 688 VALUE rb_io_gets_internal(VALUE io);
 689
 690 static rb_parser_string_t *
 691 lex_io_gets(struct parser_params *p, rb_parser_input_data input, int line_count)
 692 {
 693     VALUE io = (VALUE)input;
 694     VALUE line = rb_io_gets_internal(io);
 695     if (NIL_P(line)) return 0;
 696     return rb_str_to_parser_string(p, line);
 697 }
 698
 699 static rb_parser_string_t *
 700 lex_gets_array(struct parser_params *p, rb_parser_input_data data, int index)
 701 {
 702     VALUE array = (VALUE)data;
 703     VALUE str = rb_ary_entry(array, index);
 704     if (!NIL_P(str)) {
 705         StringValue(str);
 706         if (!rb_enc_asciicompat(rb_enc_get(str))) {
 707             rb_raise(rb_eArgError, "invalid source encoding");
 708         }
 709         return rb_str_to_parser_string(p, str);
 710     }
 711     else {
 712         return 0;
 713     }
 714 }
 715
 716 static rb_ast_t*
 717 parser_compile_file_path(struct ruby_parser *parser, VALUE fname, VALUE file, int start)
 718 {
 719     parser->type = lex_type_io;
 720     parser->data.lex_io.file = file;
 721
 722     return parser_compile(parser->parser_params, lex_io_gets, fname, (rb_parser_input_data)file, start);
 723 }
 724
 725 static rb_ast_t*
 726 parser_compile_array(struct ruby_parser *parser, VALUE fname, VALUE array, int start)
 727 {
 728     parser->type = lex_type_array;
 729     parser->data.lex_array.ary = array;
 730
 731     return parser_compile(parser->parser_params, lex_gets_array, fname, (rb_parser_input_data)array, start);
 732 }
 733
 734 static rb_ast_t*
 735 parser_compile_generic(struct ruby_parser *parser, rb_parser_lex_gets_func *lex_gets, VALUE fname, VALUE input, int start)
 736 {
 737     parser->type = lex_type_generic;
 738
 739     return parser_compile(parser->parser_params, lex_gets, fname, (rb_parser_input_data)input, start);
 740 }
 741
 742 static void
 743 ast_free(void *ptr)
 744 {
 745     rb_ast_t *ast = (rb_ast_t *)ptr;
 746     rb_ast_free(ast);
 747 }
 748
 749 static const rb_data_type_t ast_data_type = {
 750     "AST",
 751     {
 752         NULL,
 753         ast_free,
 754         NULL, // No dsize() because this object does not appear in ObjectSpace.
 755     },
 756     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
 757 };
 758
 759 static VALUE
 760 ast_alloc(void)
 761 {
 762     return TypedData_Wrap_Struct(0, &ast_data_type, NULL);
 763 }
 764
 765 VALUE
 766 rb_parser_compile_file_path(VALUE vparser, VALUE fname, VALUE file, int start)
 767 {
 768     struct ruby_parser *parser;
 769     VALUE ast_value = ast_alloc();
 770
 771     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 772     DATA_PTR(ast_value) = parser_compile_file_path(parser, fname, file, start);
 773     RB_GC_GUARD(vparser);
 774
 775     return ast_value;
 776 }
 777
 778 VALUE
 779 rb_parser_compile_array(VALUE vparser, VALUE fname, VALUE array, int start)
 780 {
 781     struct ruby_parser *parser;
 782     VALUE ast_value = ast_alloc();
 783
 784     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 785     DATA_PTR(ast_value) = parser_compile_array(parser, fname, array, start);
 786     RB_GC_GUARD(vparser);
 787
 788     return ast_value;
 789 }
 790
 791 VALUE
 792 rb_parser_compile_generic(VALUE vparser, rb_parser_lex_gets_func *lex_gets, VALUE fname, VALUE input, int start)
 793 {
 794     struct ruby_parser *parser;
 795     VALUE ast_value = ast_alloc();
 796
 797     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 798     DATA_PTR(ast_value) = parser_compile_generic(parser, lex_gets, fname, input, start);
 799     RB_GC_GUARD(vparser);
 800
 801     return ast_value;
 802 }
 803
 804 VALUE
 805 rb_parser_compile_string(VALUE vparser, const char *f, VALUE s, int line)
 806 {
 807     struct ruby_parser *parser;
 808     VALUE ast_value = ast_alloc();
 809
 810     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 811     DATA_PTR(ast_value) = parser_compile_string(parser, f, s, line);
 812     RB_GC_GUARD(vparser);
 813
 814     return ast_value;
 815 }
 816
 817 VALUE
 818 rb_parser_compile_string_path(VALUE vparser, VALUE f, VALUE s, int line)
 819 {
 820     struct ruby_parser *parser;
 821     VALUE ast_value = ast_alloc();
 822
 823     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 824     DATA_PTR(ast_value) = parser_compile_string_path(parser, f, s, line);
 825     RB_GC_GUARD(vparser);
 826
 827     return ast_value;
 828 }
 829
 830 VALUE
 831 rb_parser_encoding(VALUE vparser)
 832 {
 833     struct ruby_parser *parser;
 834
 835     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 836     return rb_enc_from_encoding(rb_ruby_parser_encoding(parser->parser_params));
 837 }
 838
 839 VALUE
 840 rb_parser_end_seen_p(VALUE vparser)
 841 {
 842     struct ruby_parser *parser;
 843
 844     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 845     return RBOOL(rb_ruby_parser_end_seen_p(parser->parser_params));
 846 }
 847
 848 VALUE
 849 rb_parser_set_yydebug(VALUE vparser, VALUE flag)
 850 {
 851     struct ruby_parser *parser;
 852
 853     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 854     rb_ruby_parser_set_yydebug(parser->parser_params, RTEST(flag));
 855     return flag;
 856 }
 857
 858 void
 859 rb_set_script_lines_for(VALUE vparser, VALUE path)
 860 {
 861     struct ruby_parser *parser;
 862     VALUE hash;
 863     ID script_lines;
 864     CONST_ID(script_lines, "SCRIPT_LINES__");
 865     if (!rb_const_defined_at(rb_cObject, script_lines)) return;
 866     hash = rb_const_get_at(rb_cObject, script_lines);
 867     if (RB_TYPE_P(hash, T_HASH)) {
 868         rb_hash_aset(hash, path, Qtrue);
 869         TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 870         rb_ruby_parser_set_script_lines(parser->parser_params);
 871     }
 872 }
 873
 874 VALUE
 875 rb_parser_build_script_lines_from(rb_parser_ary_t *lines)
 876 {
 877     int i;
 878     if (!lines) return Qnil;
 879     if (lines->data_type != PARSER_ARY_DATA_SCRIPT_LINE) {
 880         rb_bug("unexpected rb_parser_ary_data_type (%d) for script lines", lines->data_type);
 881     }
 882     VALUE script_lines = rb_ary_new_capa(lines->len);
 883     for (i = 0; i < lines->len; i++) {
 884         rb_parser_string_t *str = (rb_parser_string_t *)lines->data[i];
 885         rb_ary_push(script_lines, rb_enc_str_new(str->ptr, str->len, str->enc));
 886     }
 887     return script_lines;
 888 }
 889
 890 VALUE
 891 rb_str_new_parser_string(rb_parser_string_t *str)
 892 {
 893     VALUE string = rb_enc_literal_str(str->ptr, str->len, str->enc);
 894     rb_enc_str_coderange(string);
 895     return string;
 896 }
 897
 898 VALUE
 899 rb_str_new_mutable_parser_string(rb_parser_string_t *str)
 900 {
 901     return rb_enc_str_new(str->ptr, str->len, str->enc);
 902 }
 903
 904 static VALUE
 905 negative_numeric(VALUE val)
 906 {
 907     if (FIXNUM_P(val)) {
 908         return LONG2FIX(-FIX2LONG(val));
 909     }
 910     if (SPECIAL_CONST_P(val)) {
 911 #if USE_FLONUM
 912         if (FLONUM_P(val)) {
 913             return DBL2NUM(-RFLOAT_VALUE(val));
 914         }
 915 #endif
 916         goto unknown;
 917     }
 918     switch (BUILTIN_TYPE(val)) {
 919       case T_BIGNUM:
 920         BIGNUM_NEGATE(val);
 921         val = rb_big_norm(val);
 922         break;
 923       case T_RATIONAL:
 924         RATIONAL_SET_NUM(val, negative_numeric(RRATIONAL(val)->num));
 925         break;
 926       case T_COMPLEX:
 927         RCOMPLEX_SET_REAL(val, negative_numeric(RCOMPLEX(val)->real));
 928         RCOMPLEX_SET_IMAG(val, negative_numeric(RCOMPLEX(val)->imag));
 929         break;
 930       case T_FLOAT:
 931         val = DBL2NUM(-RFLOAT_VALUE(val));
 932         break;
 933       unknown:
 934       default:
 935         rb_bug("unknown literal type (%s) passed to negative_numeric",
 936                rb_builtin_class_name(val));
 937         break;
 938     }
 939     return val;
 940 }
 941
 942 static VALUE
 943 integer_value(const char *val, int base)
 944 {
 945     return rb_cstr_to_inum(val, base, FALSE);
 946 }
 947
 948 static VALUE
 949 rational_value(const char *node_val, int base, int seen_point)
 950 {
 951     VALUE lit;
 952     char* val = strdup(node_val);
 953     if (seen_point > 0) {
 954         int len = (int)(strlen(val));
 955         char *point = &val[seen_point];
 956         size_t fraclen = len-seen_point-1;
 957         memmove(point, point+1, fraclen+1);
 958
 959         lit = rb_rational_new(integer_value(val, base), rb_int_positive_pow(10, fraclen));
 960     }
 961     else {
 962         lit = rb_rational_raw1(integer_value(val, base));
 963     }
 964
 965     free(val);
 966
 967     return lit;
 968 }
 969
 970 VALUE
 971 rb_node_integer_literal_val(const NODE *n)
 972 {
 973     const rb_node_integer_t *node = RNODE_INTEGER(n);
 974     VALUE val = integer_value(node->val, node->base);
 975     if (node->minus) {
 976         val = negative_numeric(val);
 977     }
 978     return val;
 979 }
 980
 981 VALUE
 982 rb_node_float_literal_val(const NODE *n)
 983 {
 984     const rb_node_float_t *node = RNODE_FLOAT(n);
 985     double d = strtod(node->val, 0);
 986     if (node->minus) {
 987         d = -d;
 988     }
 989     VALUE val = DBL2NUM(d);
 990     return val;
 991 }
 992
 993 VALUE
 994 rb_node_rational_literal_val(const NODE *n)
 995 {
 996     VALUE lit;
 997     const rb_node_rational_t *node = RNODE_RATIONAL(n);
 998
 999     lit = rational_value(node->val, node->base, node->seen_point);
1000
1001     if (node->minus) {
1002         lit = negative_numeric(lit);
1003     }
1004
1005     return lit;
1006 }
1007
1008 VALUE
1009 rb_node_imaginary_literal_val(const NODE *n)
1010 {
1011     VALUE lit;
1012     const rb_node_imaginary_t *node = RNODE_IMAGINARY(n);
1013
1014     enum rb_numeric_type type = node->type;
1015
1016     switch (type) {
1017       case integer_literal:
1018         lit = integer_value(node->val, node->base);
1019         break;
1020       case float_literal:{
1021         double d = strtod(node->val, 0);
1022         lit = DBL2NUM(d);
1023         break;
1024       }
1025       case rational_literal:
1026         lit = rational_value(node->val, node->base, node->seen_point);
1027         break;
1028       default:
1029         rb_bug("unreachable");
1030     }
1031
1032     lit = rb_complex_raw(INT2FIX(0), lit);
1033
1034     if (node->minus) {
1035         lit = negative_numeric(lit);
1036     }
1037     return lit;
1038 }
1039
1040 VALUE
1041 rb_node_str_string_val(const NODE *node)
1042 {
1043     rb_parser_string_t *str = RNODE_STR(node)->string;
1044     return rb_str_new_parser_string(str);
1045 }
1046
1047 VALUE
1048 rb_node_sym_string_val(const NODE *node)
1049 {
1050     rb_parser_string_t *str = RNODE_SYM(node)->string;
1051     return ID2SYM(rb_intern3(str->ptr, str->len, str->enc));
1052 }
1053
1054 VALUE
1055 rb_node_dstr_string_val(const NODE *node)
1056 {
1057     rb_parser_string_t *str = RNODE_DSTR(node)->string;
1058     return str ? rb_str_new_parser_string(str) : Qnil;
1059 }
1060
1061 VALUE
1062 rb_node_dregx_string_val(const NODE *node)
1063 {
1064     rb_parser_string_t *str = RNODE_DREGX(node)->string;
1065     return rb_str_new_parser_string(str);
1066 }
1067
1068 VALUE
1069 rb_node_regx_string_val(const NODE *node)
1070 {
1071     rb_node_regx_t *node_reg = RNODE_REGX(node);
1072     rb_parser_string_t *string = node_reg->string;
1073     VALUE str = rb_enc_str_new(string->ptr, string->len, string->enc);
1074
1075     return rb_reg_compile(str, node_reg->options, NULL, 0);
1076 }
1077
1078 VALUE
1079 rb_node_line_lineno_val(const NODE *node)
1080 {
1081     return INT2FIX(node->nd_loc.beg_pos.lineno);
1082 }
1083
1084 VALUE
1085 rb_node_file_path_val(const NODE *node)
1086 {
1087     return rb_str_new_parser_string(RNODE_FILE(node)->path);
1088 }
1089
1090 VALUE
1091 rb_node_encoding_val(const NODE *node)
1092 {
1093     return rb_enc_from_encoding(RNODE_ENCODING(node)->enc);
1094 }
1095
1096 static void
1097 parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *lines)
1098 {
1099     VALUE hash, script_lines;
1100     ID script_lines_id;
1101     if (NIL_P(path) || !lines) return;
1102     CONST_ID(script_lines_id, "SCRIPT_LINES__");
1103     if (!rb_const_defined_at(rb_cObject, script_lines_id)) return;
1104     hash = rb_const_get_at(rb_cObject, script_lines_id);
1105     if (!RB_TYPE_P(hash, T_HASH)) return;
1106     if (rb_hash_lookup(hash, path) == Qnil) return;
1107     script_lines = rb_parser_build_script_lines_from(lines);
1108     rb_hash_aset(hash, path, script_lines);
1109 }
1110
1111 VALUE
1112 rb_ruby_ast_new(const NODE *const root)
1113 {
1114     rb_ast_t *ast;
1115     VALUE ast_value = TypedData_Make_Struct(0, rb_ast_t, &ast_data_type, ast);
1116 #ifdef UNIVERSAL_PARSER
1117     ast->config = &rb_global_parser_config;
1118 #endif
1119     ast->body = (rb_ast_body_t){
1120         .root = root,
1121         .frozen_string_literal = -1,
1122         .coverage_enabled = -1,
1123         .script_lines = NULL,
1124         .line_count = 0,
1125     };
1126     return ast_value;
1127 }
1128
1129 rb_ast_t *
1130 rb_ruby_ast_data_get(VALUE ast_value)
1131 {
1132     rb_ast_t *ast;
1133     if (NIL_P(ast_value)) return NULL;
1134     TypedData_Get_Struct(ast_value, rb_ast_t, &ast_data_type, ast);
1135     return ast;
1136 }