ruby_parser.c

   1 /* This is a wrapper for parse.y */
   2
   3 #include "internal/parse.h"
   4 #include "internal/re.h"
   5 #include "internal/ruby_parser.h"
   6
   7 #include "node.h"
   8 #include "rubyparser.h"
   9 #include "internal/error.h"
  10
  11 #ifdef UNIVERSAL_PARSER
  12
  13 #include "internal.h"
  14 #include "internal/array.h"
  15 #include "internal/bignum.h"
  16 #include "internal/compile.h"
  17 #include "internal/complex.h"
  18 #include "internal/encoding.h"
  19 #include "internal/gc.h"
  20 #include "internal/hash.h"
  21 #include "internal/io.h"
  22 #include "internal/rational.h"
  23 #include "internal/re.h"
  24 #include "internal/string.h"
  25 #include "internal/symbol.h"
  26 #include "internal/thread.h"
  27
  28 #include "ruby/ractor.h"
  29 #include "ruby/ruby.h"
  30 #include "ruby/util.h"
  31 #include "internal.h"
  32 #include "vm_core.h"
  33 #include "symbol.h"
  34
  35 static int
  36 is_ascii_string2(VALUE str)
  37 {
  38     return is_ascii_string(str);
  39 }
  40
  41 RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 6, 0)
  42 static VALUE
  43 syntax_error_append(VALUE exc, VALUE file, int line, int column,
  44                        void *enc, const char *fmt, va_list args)
  45 {
  46     return rb_syntax_error_append(exc, file, line, column, (rb_encoding *)enc, fmt, args);
  47 }
  48
  49 static int
  50 local_defined(ID id, const void *p)
  51 {
  52     return rb_local_defined(id, (const rb_iseq_t *)p);
  53 }
  54
  55 static int
  56 dvar_defined(ID id, const void *p)
  57 {
  58     return rb_dvar_defined(id, (const rb_iseq_t *)p);
  59 }
  60
  61 static int
  62 is_usascii_enc(void *enc)
  63 {
  64     return rb_is_usascii_enc((rb_encoding *)enc);
  65 }
  66
  67 static int
  68 is_local_id2(ID id)
  69 {
  70     return is_local_id(id);
  71 }
  72
  73 static int
  74 is_attrset_id2(ID id)
  75 {
  76     return is_attrset_id(id);
  77 }
  78
  79 static int
  80 is_notop_id2(ID id)
  81 {
  82     return is_notop_id(id);
  83 }
  84
  85 static VALUE
  86 enc_str_new(const char *ptr, long len, void *enc)
  87 {
  88     return rb_enc_str_new(ptr, len, (rb_encoding *)enc);
  89 }
  90
  91 static int
  92 enc_isalnum(OnigCodePoint c, void *enc)
  93 {
  94     return rb_enc_isalnum(c, (rb_encoding *)enc);
  95 }
  96
  97 static int
  98 enc_precise_mbclen(const char *p, const char *e, void *enc)
  99 {
 100     return rb_enc_precise_mbclen(p, e, (rb_encoding *)enc);
 101 }
 102
 103 static int
 104 mbclen_charfound_p(int len)
 105 {
 106     return MBCLEN_CHARFOUND_P(len);
 107 }
 108
 109 static int
 110 mbclen_charfound_len(int len)
 111 {
 112     return MBCLEN_CHARFOUND_LEN(len);
 113 }
 114
 115 static const char *
 116 enc_name(void *enc)
 117 {
 118     return rb_enc_name((rb_encoding *)enc);
 119 }
 120
 121 static char *
 122 enc_prev_char(const char *s, const char *p, const char *e, void *enc)
 123 {
 124     return rb_enc_prev_char(s, p, e, (rb_encoding *)enc);
 125 }
 126
 127 static void *
 128 enc_get(VALUE obj)
 129 {
 130     return (void *)rb_enc_get(obj);
 131 }
 132
 133 static int
 134 enc_asciicompat(void *enc)
 135 {
 136     return rb_enc_asciicompat((rb_encoding *)enc);
 137 }
 138
 139 static void *
 140 utf8_encoding(void)
 141 {
 142     return (void *)rb_utf8_encoding();
 143 }
 144
 145 static VALUE
 146 enc_associate(VALUE obj, void *enc)
 147 {
 148     return rb_enc_associate(obj, (rb_encoding *)enc);
 149 }
 150
 151 static void *
 152 ascii8bit_encoding(void)
 153 {
 154     return (void *)rb_ascii8bit_encoding();
 155 }
 156
 157 static int
 158 enc_codelen(int c, void *enc)
 159 {
 160     return rb_enc_codelen(c, (rb_encoding *)enc);
 161 }
 162
 163 static int
 164 enc_mbcput(unsigned int c, void *buf, void *enc)
 165 {
 166     return rb_enc_mbcput(c, buf, (rb_encoding *)enc);
 167 }
 168
 169 static int
 170 enc_mbclen(const char *p, const char *e, void *enc)
 171 {
 172     return rb_enc_mbclen(p, e, (rb_encoding *)enc);
 173 }
 174
 175 static void *
 176 enc_from_index(int idx)
 177 {
 178     return (void *)rb_enc_from_index(idx);
 179 }
 180
 181 static int
 182 enc_isspace(OnigCodePoint c, void *enc)
 183 {
 184     return rb_enc_isspace(c, (rb_encoding *)enc);
 185 }
 186
 187 static ID
 188 intern3(const char *name, long len, void *enc)
 189 {
 190     return rb_intern3(name, len, (rb_encoding *)enc);
 191 }
 192
 193 static void *
 194 usascii_encoding(void)
 195 {
 196     return (void *)rb_usascii_encoding();
 197 }
 198
 199 static int
 200 enc_symname_type(const char *name, long len, void *enc, unsigned int allowed_attrset)
 201 {
 202     return rb_enc_symname_type(name, len, (rb_encoding *)enc, allowed_attrset);
 203 }
 204
 205 typedef struct {
 206     struct parser_params *parser;
 207     rb_encoding *enc;
 208     NODE *succ_block;
 209     const rb_code_location_t *loc;
 210 } reg_named_capture_assign_t;
 211
 212 static int
 213 reg_named_capture_assign_iter(const OnigUChar *name, const OnigUChar *name_end,
 214           int back_num, int *back_refs, OnigRegex regex, void *arg0)
 215 {
 216     reg_named_capture_assign_t *arg = (reg_named_capture_assign_t*)arg0;
 217     struct parser_params* p = arg->parser;
 218     rb_encoding *enc = arg->enc;
 219     const rb_code_location_t *loc = arg->loc;
 220     long len = name_end - name;
 221     const char *s = (const char *)name;
 222
 223     return rb_reg_named_capture_assign_iter_impl(p, s, len, (void *)enc, &arg->succ_block, loc);
 224 }
 225
 226 static NODE *
 227 reg_named_capture_assign(struct parser_params* p, VALUE regexp, const rb_code_location_t *loc)
 228 {
 229     reg_named_capture_assign_t arg;
 230
 231     arg.parser = p;
 232     arg.enc = rb_enc_get(regexp);
 233     arg.succ_block = 0;
 234     arg.loc = loc;
 235     onig_foreach_name(RREGEXP_PTR(regexp), reg_named_capture_assign_iter, &arg);
 236
 237     if (!arg.succ_block) return 0;
 238     return RNODE_BLOCK(arg.succ_block)->nd_next;
 239 }
 240
 241 static int
 242 rtest(VALUE obj)
 243 {
 244     return (int)RB_TEST(obj);
 245 }
 246
 247 static int
 248 nil_p(VALUE obj)
 249 {
 250     return (int)NIL_P(obj);
 251 }
 252
 253 static VALUE
 254 syntax_error_new(void)
 255 {
 256     return rb_class_new_instance(0, 0, rb_eSyntaxError);
 257 }
 258
 259 static void *
 260 memmove2(void *dest, const void *src, size_t t, size_t n)
 261 {
 262     return memmove(dest, src, rbimpl_size_mul_or_raise(t, n));
 263 }
 264
 265 static void *
 266 nonempty_memcpy(void *dest, const void *src, size_t t, size_t n)
 267 {
 268     return ruby_nonempty_memcpy(dest, src, rbimpl_size_mul_or_raise(t, n));
 269 }
 270
 271 static VALUE
 272 ruby_verbose2(void)
 273 {
 274     return ruby_verbose;
 275 }
 276
 277 static int *
 278 rb_errno_ptr2(void)
 279 {
 280     return rb_errno_ptr();
 281 }
 282
 283 static void *
 284 zalloc(size_t elemsiz)
 285 {
 286     return ruby_xcalloc(1, elemsiz);
 287 }
 288
 289 static void
 290 gc_guard(VALUE obj)
 291 {
 292     RB_GC_GUARD(obj);
 293 }
 294
 295 static VALUE
 296 arg_error(void)
 297 {
 298     return rb_eArgError;
 299 }
 300
 301 static VALUE
 302 static_id2sym(ID id)
 303 {
 304     return (((VALUE)(id)<<RUBY_SPECIAL_SHIFT)|SYMBOL_FLAG);
 305 }
 306
 307 static long
 308 str_coderange_scan_restartable(const char *s, const char *e, void *enc, int *cr)
 309 {
 310     return rb_str_coderange_scan_restartable(s, e, (rb_encoding *)enc, cr);
 311 }
 312
 313 static int
 314 enc_mbminlen(void *enc)
 315 {
 316     return rb_enc_mbminlen((rb_encoding *)enc);
 317 }
 318
 319 static bool
 320 enc_isascii(OnigCodePoint c, void *enc)
 321 {
 322     return rb_enc_isascii(c, (rb_encoding *)enc);
 323 }
 324
 325 static OnigCodePoint
 326 enc_mbc_to_codepoint(const char *p, const char *e, void *enc)
 327 {
 328     const OnigUChar *up = RBIMPL_CAST((const OnigUChar *)p);
 329     const OnigUChar *ue = RBIMPL_CAST((const OnigUChar *)e);
 330
 331     return ONIGENC_MBC_TO_CODE((rb_encoding *)enc, up, ue);
 332 }
 333
 334 extern VALUE rb_eArgError;
 335
 336 static const rb_parser_config_t rb_global_parser_config = {
 337     .malloc = ruby_xmalloc,
 338     .calloc = ruby_xcalloc,
 339     .realloc = ruby_xrealloc,
 340     .free = ruby_xfree,
 341     .alloc_n = ruby_xmalloc2,
 342     .alloc = ruby_xmalloc,
 343     .realloc_n = ruby_xrealloc2,
 344     .zalloc = zalloc,
 345     .rb_memmove = memmove2,
 346     .nonempty_memcpy = nonempty_memcpy,
 347     .xmalloc_mul_add = rb_xmalloc_mul_add,
 348
 349     .compile_callback = rb_suppress_tracing,
 350     .reg_named_capture_assign = reg_named_capture_assign,
 351
 352     .attr_get = rb_attr_get,
 353
 354     .ary_new = rb_ary_new,
 355     .ary_push = rb_ary_push,
 356     .ary_new_from_args = rb_ary_new_from_args,
 357     .ary_unshift = rb_ary_unshift,
 358
 359     .make_temporary_id = rb_make_temporary_id,
 360     .is_local_id = is_local_id2,
 361     .is_attrset_id = is_attrset_id2,
 362     .is_global_name_punct = is_global_name_punct,
 363     .id_type = id_type,
 364     .id_attrset = rb_id_attrset,
 365     .intern = rb_intern,
 366     .intern2 = rb_intern2,
 367     .intern3 = intern3,
 368     .intern_str = rb_intern_str,
 369     .is_notop_id = is_notop_id2,
 370     .enc_symname_type = enc_symname_type,
 371     .id2name = rb_id2name,
 372     .id2str = rb_id2str,
 373     .id2sym = rb_id2sym,
 374     .sym2id = rb_sym2id,
 375
 376     .str_catf = rb_str_catf,
 377     .str_cat_cstr = rb_str_cat_cstr,
 378     .str_modify = rb_str_modify,
 379     .str_set_len = rb_str_set_len,
 380     .str_cat = rb_str_cat,
 381     .str_resize = rb_str_resize,
 382     .str_new = rb_str_new,
 383     .str_new_cstr = rb_str_new_cstr,
 384     .str_to_interned_str = rb_str_to_interned_str,
 385     .is_ascii_string = is_ascii_string2,
 386     .enc_str_new = enc_str_new,
 387     .str_vcatf = rb_str_vcatf,
 388     .string_value_cstr = rb_string_value_cstr,
 389     .rb_sprintf = rb_sprintf,
 390     .rstring_ptr = RSTRING_PTR,
 391     .rstring_end = RSTRING_END,
 392     .rstring_len = RSTRING_LEN,
 393     .obj_as_string = rb_obj_as_string,
 394
 395     .int2num = rb_int2num_inline,
 396
 397     .stderr_tty_p = rb_stderr_tty_p,
 398     .write_error_str = rb_write_error_str,
 399     .io_write = rb_io_write,
 400     .io_flush = rb_io_flush,
 401     .io_puts = rb_io_puts,
 402
 403     .debug_output_stdout = rb_ractor_stdout,
 404     .debug_output_stderr = rb_ractor_stderr,
 405
 406     .is_usascii_enc = is_usascii_enc,
 407     .enc_isalnum = enc_isalnum,
 408     .enc_precise_mbclen = enc_precise_mbclen,
 409     .mbclen_charfound_p = mbclen_charfound_p,
 410     .mbclen_charfound_len = mbclen_charfound_len,
 411     .enc_name = enc_name,
 412     .enc_prev_char = enc_prev_char,
 413     .enc_get = enc_get,
 414     .enc_asciicompat = enc_asciicompat,
 415     .utf8_encoding = utf8_encoding,
 416     .enc_associate = enc_associate,
 417     .ascii8bit_encoding = ascii8bit_encoding,
 418     .enc_codelen = enc_codelen,
 419     .enc_mbcput = enc_mbcput,
 420     .enc_mbclen = enc_mbclen,
 421     .enc_find_index = rb_enc_find_index,
 422     .enc_from_index = enc_from_index,
 423     .enc_isspace = enc_isspace,
 424     .enc_coderange_7bit = ENC_CODERANGE_7BIT,
 425     .enc_coderange_unknown = ENC_CODERANGE_UNKNOWN,
 426     .usascii_encoding = usascii_encoding,
 427     .enc_coderange_broken = ENC_CODERANGE_BROKEN,
 428     .enc_mbminlen = enc_mbminlen,
 429     .enc_isascii = enc_isascii,
 430     .enc_mbc_to_codepoint = enc_mbc_to_codepoint,
 431
 432     .local_defined = local_defined,
 433     .dvar_defined = dvar_defined,
 434
 435     .syntax_error_append = syntax_error_append,
 436     .raise = rb_raise,
 437     .syntax_error_new = syntax_error_new,
 438
 439     .errinfo = rb_errinfo,
 440     .set_errinfo = rb_set_errinfo,
 441     .exc_raise = rb_exc_raise,
 442     .make_exception = rb_make_exception,
 443
 444     .sized_xfree = ruby_sized_xfree,
 445     .sized_realloc_n = ruby_sized_realloc_n,
 446     .gc_guard = gc_guard,
 447     .gc_mark = rb_gc_mark,
 448
 449     .reg_compile = rb_reg_compile,
 450     .reg_check_preprocess = rb_reg_check_preprocess,
 451     .memcicmp = rb_memcicmp,
 452
 453     .compile_warn = rb_compile_warn,
 454     .compile_warning = rb_compile_warning,
 455     .bug = rb_bug,
 456     .fatal = rb_fatal,
 457     .verbose = ruby_verbose2,
 458     .errno_ptr = rb_errno_ptr2,
 459
 460     .make_backtrace = rb_make_backtrace,
 461
 462     .scan_hex = ruby_scan_hex,
 463     .scan_oct = ruby_scan_oct,
 464     .scan_digits = ruby_scan_digits,
 465     .strtod = ruby_strtod,
 466
 467     .rtest = rtest,
 468     .nil_p = nil_p,
 469     .qnil = Qnil,
 470     .qfalse = Qfalse,
 471     .eArgError = arg_error,
 472     .long2int = rb_long2int,
 473
 474     /* For Ripper */
 475     .static_id2sym = static_id2sym,
 476     .str_coderange_scan_restartable = str_coderange_scan_restartable,
 477 };
 478 #endif
 479
 480 enum lex_type {
 481     lex_type_str,
 482     lex_type_io,
 483     lex_type_array,
 484     lex_type_generic,
 485 };
 486
 487 struct ruby_parser {
 488     rb_parser_t *parser_params;
 489     enum lex_type type;
 490     union {
 491         struct lex_pointer_string lex_str;
 492         struct {
 493             VALUE file;
 494         } lex_io;
 495         struct {
 496             VALUE ary;
 497         } lex_array;
 498     } data;
 499 };
 500
 501 static void
 502 parser_mark(void *ptr)
 503 {
 504     struct ruby_parser *parser = (struct ruby_parser*)ptr;
 505     rb_ruby_parser_mark(parser->parser_params);
 506
 507     switch (parser->type) {
 508       case lex_type_str:
 509         rb_gc_mark(parser->data.lex_str.str);
 510         break;
 511       case lex_type_io:
 512         rb_gc_mark(parser->data.lex_io.file);
 513         break;
 514       case lex_type_array:
 515         rb_gc_mark(parser->data.lex_array.ary);
 516         break;
 517       case lex_type_generic:
 518         /* noop. Caller of rb_parser_compile_generic should mark the objects. */
 519         break;
 520     }
 521 }
 522
 523 static void
 524 parser_free(void *ptr)
 525 {
 526     struct ruby_parser *parser = (struct ruby_parser*)ptr;
 527     rb_ruby_parser_free(parser->parser_params);
 528     xfree(parser);
 529 }
 530
 531 static size_t
 532 parser_memsize(const void *ptr)
 533 {
 534     struct ruby_parser *parser = (struct ruby_parser*)ptr;
 535     return rb_ruby_parser_memsize(parser->parser_params);
 536 }
 537
 538 static const rb_data_type_t ruby_parser_data_type = {
 539     "parser",
 540     {
 541         parser_mark,
 542         parser_free,
 543         parser_memsize,
 544     },
 545     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
 546 };
 547
 548 #ifdef UNIVERSAL_PARSER
 549 const rb_parser_config_t *
 550 rb_ruby_parser_config(void)
 551 {
 552     return &rb_global_parser_config;
 553 }
 554
 555 rb_parser_t *
 556 rb_parser_params_new(void)
 557 {
 558     return rb_ruby_parser_new(&rb_global_parser_config);
 559 }
 560 #else
 561 rb_parser_t *
 562 rb_parser_params_new(void)
 563 {
 564     return rb_ruby_parser_new();
 565 }
 566 #endif /* UNIVERSAL_PARSER */
 567
 568 VALUE
 569 rb_parser_new(void)
 570 {
 571     struct ruby_parser *parser;
 572     rb_parser_t *parser_params;
 573
 574     /*
 575      * Create parser_params ahead of vparser because
 576      * rb_ruby_parser_new can run GC so if create vparser
 577      * first, parser_mark tries to mark not initialized parser_params.
 578      */
 579     parser_params = rb_parser_params_new();
 580     VALUE vparser = TypedData_Make_Struct(0, struct ruby_parser,
 581                                          &ruby_parser_data_type, parser);
 582     parser->parser_params = parser_params;
 583
 584     return vparser;
 585 }
 586
 587 void
 588 rb_parser_set_options(VALUE vparser, int print, int loop, int chomp, int split)
 589 {
 590     struct ruby_parser *parser;
 591
 592     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 593     rb_ruby_parser_set_options(parser->parser_params, print, loop, chomp, split);
 594 }
 595
 596 VALUE
 597 rb_parser_set_context(VALUE vparser, const struct rb_iseq_struct *base, int main)
 598 {
 599     struct ruby_parser *parser;
 600
 601     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 602     rb_ruby_parser_set_context(parser->parser_params, base, main);
 603     return vparser;
 604 }
 605
 606 void
 607 rb_parser_set_script_lines(VALUE vparser)
 608 {
 609     struct ruby_parser *parser;
 610
 611     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 612     rb_ruby_parser_set_script_lines(parser->parser_params);
 613 }
 614
 615 void
 616 rb_parser_error_tolerant(VALUE vparser)
 617 {
 618     struct ruby_parser *parser;
 619
 620     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 621     rb_ruby_parser_error_tolerant(parser->parser_params);
 622 }
 623
 624 void
 625 rb_parser_keep_tokens(VALUE vparser)
 626 {
 627     struct ruby_parser *parser;
 628
 629     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 630     rb_ruby_parser_keep_tokens(parser->parser_params);
 631 }
 632
 633 VALUE
 634 rb_parser_lex_get_str(struct lex_pointer_string *ptr_str)
 635 {
 636     char *beg, *end, *start;
 637     long len;
 638     VALUE s = ptr_str->str;
 639
 640     beg = RSTRING_PTR(s);
 641     len = RSTRING_LEN(s);
 642     start = beg;
 643     if (ptr_str->ptr) {
 644         if (len == ptr_str->ptr) return Qnil;
 645         beg += ptr_str->ptr;
 646         len -= ptr_str->ptr;
 647     }
 648     end = memchr(beg, '\n', len);
 649     if (end) len = ++end - beg;
 650     ptr_str->ptr += len;
 651     return rb_str_subseq(s, beg - start, len);
 652 }
 653
 654 static VALUE
 655 lex_get_str(struct parser_params *p, rb_parser_input_data input, int line_count)
 656 {
 657     return rb_parser_lex_get_str((struct lex_pointer_string *)input);
 658 }
 659
 660 static void parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *lines);
 661
 662 static rb_ast_t*
 663 parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, VALUE fname, rb_parser_input_data input, int line)
 664 {
 665     rb_ast_t *ast;
 666     const char *ptr = 0;
 667     long len = 0;
 668     rb_encoding *enc = 0;
 669
 670     if (!NIL_P(fname)) {
 671         StringValueCStr(fname);
 672         ptr = RSTRING_PTR(fname);
 673         len = RSTRING_LEN(fname);
 674         enc = rb_enc_get(fname);
 675     }
 676
 677     ast = rb_parser_compile(p, gets, ptr, len, enc, input, line);
 678     parser_aset_script_lines_for(fname, ast->body.script_lines);
 679     return ast;
 680 }
 681
 682 static rb_ast_t*
 683 parser_compile_string0(struct ruby_parser *parser, VALUE fname, VALUE s, int line)
 684 {
 685     VALUE str = rb_str_new_frozen(s);
 686
 687     parser->type = lex_type_str;
 688     parser->data.lex_str.str = str;
 689     parser->data.lex_str.ptr = 0;
 690
 691     return parser_compile(parser->parser_params, lex_get_str, fname, (rb_parser_input_data)&parser->data, line);
 692 }
 693
 694 static rb_encoding *
 695 must_be_ascii_compatible(VALUE s)
 696 {
 697     rb_encoding *enc = rb_enc_get(s);
 698     if (!rb_enc_asciicompat(enc)) {
 699         rb_raise(rb_eArgError, "invalid source encoding");
 700     }
 701     return enc;
 702 }
 703
 704 static rb_ast_t*
 705 parser_compile_string_path(struct ruby_parser *parser, VALUE f, VALUE s, int line)
 706 {
 707     must_be_ascii_compatible(s);
 708     return parser_compile_string0(parser, f, s, line);
 709 }
 710
 711 static rb_ast_t*
 712 parser_compile_string(struct ruby_parser *parser, const char *f, VALUE s, int line)
 713 {
 714     return parser_compile_string_path(parser, rb_filesystem_str_new_cstr(f), s, line);
 715 }
 716
 717 VALUE rb_io_gets_internal(VALUE io);
 718
 719 static VALUE
 720 lex_io_gets(struct parser_params *p, rb_parser_input_data input, int line_count)
 721 {
 722     VALUE io = (VALUE)input;
 723
 724     return rb_io_gets_internal(io);
 725 }
 726
 727 static VALUE
 728 lex_gets_array(struct parser_params *p, rb_parser_input_data data, int index)
 729 {
 730     VALUE array = (VALUE)data;
 731     VALUE str = rb_ary_entry(array, index);
 732     if (!NIL_P(str)) {
 733         StringValue(str);
 734         if (!rb_enc_asciicompat(rb_enc_get(str))) {
 735             rb_raise(rb_eArgError, "invalid source encoding");
 736         }
 737     }
 738     return str;
 739 }
 740
 741 static rb_ast_t*
 742 parser_compile_file_path(struct ruby_parser *parser, VALUE fname, VALUE file, int start)
 743 {
 744     parser->type = lex_type_io;
 745     parser->data.lex_io.file = file;
 746
 747     return parser_compile(parser->parser_params, lex_io_gets, fname, (rb_parser_input_data)file, start);
 748 }
 749
 750 static rb_ast_t*
 751 parser_compile_array(struct ruby_parser *parser, VALUE fname, VALUE array, int start)
 752 {
 753     parser->type = lex_type_array;
 754     parser->data.lex_array.ary = array;
 755
 756     return parser_compile(parser->parser_params, lex_gets_array, fname, (rb_parser_input_data)array, start);
 757 }
 758
 759 static rb_ast_t*
 760 parser_compile_generic(struct ruby_parser *parser, rb_parser_lex_gets_func *lex_gets, VALUE fname, VALUE input, int start)
 761 {
 762     parser->type = lex_type_generic;
 763
 764     return parser_compile(parser->parser_params, lex_gets, fname, (rb_parser_input_data)input, start);
 765 }
 766
 767 static void
 768 ast_free(void *ptr)
 769 {
 770     rb_ast_t *ast = (rb_ast_t *)ptr;
 771     rb_ast_free(ast);
 772 }
 773
 774 static const rb_data_type_t ast_data_type = {
 775     "AST",
 776     {
 777         NULL,
 778         ast_free,
 779         NULL, // No dsize() because this object does not appear in ObjectSpace.
 780     },
 781     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
 782 };
 783
 784 static VALUE
 785 ast_alloc(void)
 786 {
 787     return TypedData_Wrap_Struct(0, &ast_data_type, NULL);
 788 }
 789
 790 VALUE
 791 rb_parser_compile_file_path(VALUE vparser, VALUE fname, VALUE file, int start)
 792 {
 793     struct ruby_parser *parser;
 794     VALUE ast_value = ast_alloc();
 795
 796     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 797     DATA_PTR(ast_value) = parser_compile_file_path(parser, fname, file, start);
 798     RB_GC_GUARD(vparser);
 799
 800     return ast_value;
 801 }
 802
 803 VALUE
 804 rb_parser_compile_array(VALUE vparser, VALUE fname, VALUE array, int start)
 805 {
 806     struct ruby_parser *parser;
 807     VALUE ast_value = ast_alloc();
 808
 809     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 810     DATA_PTR(ast_value) = parser_compile_array(parser, fname, array, start);
 811     RB_GC_GUARD(vparser);
 812
 813     return ast_value;
 814 }
 815
 816 VALUE
 817 rb_parser_compile_generic(VALUE vparser, rb_parser_lex_gets_func *lex_gets, VALUE fname, VALUE input, int start)
 818 {
 819     struct ruby_parser *parser;
 820     VALUE ast_value = ast_alloc();
 821
 822     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 823     DATA_PTR(ast_value) = parser_compile_generic(parser, lex_gets, fname, input, start);
 824     RB_GC_GUARD(vparser);
 825
 826     return ast_value;
 827 }
 828
 829 VALUE
 830 rb_parser_compile_string(VALUE vparser, const char *f, VALUE s, int line)
 831 {
 832     struct ruby_parser *parser;
 833     VALUE ast_value = ast_alloc();
 834
 835     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 836     DATA_PTR(ast_value) = parser_compile_string(parser, f, s, line);
 837     RB_GC_GUARD(vparser);
 838
 839     return ast_value;
 840 }
 841
 842 VALUE
 843 rb_parser_compile_string_path(VALUE vparser, VALUE f, VALUE s, int line)
 844 {
 845     struct ruby_parser *parser;
 846     VALUE ast_value = ast_alloc();
 847
 848     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 849     DATA_PTR(ast_value) = parser_compile_string_path(parser, f, s, line);
 850     RB_GC_GUARD(vparser);
 851
 852     return ast_value;
 853 }
 854
 855 VALUE
 856 rb_parser_encoding(VALUE vparser)
 857 {
 858     struct ruby_parser *parser;
 859
 860     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 861     return rb_enc_from_encoding(rb_ruby_parser_encoding(parser->parser_params));
 862 }
 863
 864 VALUE
 865 rb_parser_end_seen_p(VALUE vparser)
 866 {
 867     struct ruby_parser *parser;
 868
 869     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 870     return RBOOL(rb_ruby_parser_end_seen_p(parser->parser_params));
 871 }
 872
 873 VALUE
 874 rb_parser_set_yydebug(VALUE vparser, VALUE flag)
 875 {
 876     struct ruby_parser *parser;
 877
 878     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 879     rb_ruby_parser_set_yydebug(parser->parser_params, RTEST(flag));
 880     return flag;
 881 }
 882
 883 void
 884 rb_set_script_lines_for(VALUE vparser, VALUE path)
 885 {
 886     struct ruby_parser *parser;
 887     VALUE hash;
 888     ID script_lines;
 889     CONST_ID(script_lines, "SCRIPT_LINES__");
 890     if (!rb_const_defined_at(rb_cObject, script_lines)) return;
 891     hash = rb_const_get_at(rb_cObject, script_lines);
 892     if (RB_TYPE_P(hash, T_HASH)) {
 893         rb_hash_aset(hash, path, Qtrue);
 894         TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 895         rb_ruby_parser_set_script_lines(parser->parser_params);
 896     }
 897 }
 898
 899 VALUE
 900 rb_parser_build_script_lines_from(rb_parser_ary_t *lines)
 901 {
 902     int i;
 903     if (!lines) return Qnil;
 904     if (lines->data_type != PARSER_ARY_DATA_SCRIPT_LINE) {
 905         rb_bug("unexpected rb_parser_ary_data_type (%d) for script lines", lines->data_type);
 906     }
 907     VALUE script_lines = rb_ary_new_capa(lines->len);
 908     for (i = 0; i < lines->len; i++) {
 909         rb_parser_string_t *str = (rb_parser_string_t *)lines->data[i];
 910         rb_ary_push(script_lines, rb_enc_str_new(str->ptr, str->len, str->enc));
 911     }
 912     return script_lines;
 913 }
 914
 915 VALUE
 916 rb_str_new_parser_string(rb_parser_string_t *str)
 917 {
 918     VALUE string = rb_enc_interned_str(str->ptr, str->len, str->enc);
 919     rb_enc_str_coderange(string);
 920     return string;
 921 }
 922
 923 VALUE
 924 rb_str_new_mutable_parser_string(rb_parser_string_t *str)
 925 {
 926     return rb_enc_str_new(str->ptr, str->len, str->enc);
 927 }
 928
 929 static VALUE
 930 negative_numeric(VALUE val)
 931 {
 932     if (FIXNUM_P(val)) {
 933         return LONG2FIX(-FIX2LONG(val));
 934     }
 935     if (SPECIAL_CONST_P(val)) {
 936 #if USE_FLONUM
 937         if (FLONUM_P(val)) {
 938             return DBL2NUM(-RFLOAT_VALUE(val));
 939         }
 940 #endif
 941         goto unknown;
 942     }
 943     switch (BUILTIN_TYPE(val)) {
 944       case T_BIGNUM:
 945         BIGNUM_NEGATE(val);
 946         val = rb_big_norm(val);
 947         break;
 948       case T_RATIONAL:
 949         RATIONAL_SET_NUM(val, negative_numeric(RRATIONAL(val)->num));
 950         break;
 951       case T_COMPLEX:
 952         RCOMPLEX_SET_REAL(val, negative_numeric(RCOMPLEX(val)->real));
 953         RCOMPLEX_SET_IMAG(val, negative_numeric(RCOMPLEX(val)->imag));
 954         break;
 955       case T_FLOAT:
 956         val = DBL2NUM(-RFLOAT_VALUE(val));
 957         break;
 958       unknown:
 959       default:
 960         rb_bug("unknown literal type (%s) passed to negative_numeric",
 961                rb_builtin_class_name(val));
 962         break;
 963     }
 964     return val;
 965 }
 966
 967 static VALUE
 968 integer_value(const char *val, int base)
 969 {
 970     return rb_cstr_to_inum(val, base, FALSE);
 971 }
 972
 973 static VALUE
 974 rational_value(const char *node_val, int base, int seen_point)
 975 {
 976     VALUE lit;
 977     char* val = strdup(node_val);
 978     if (seen_point > 0) {
 979         int len = (int)(strlen(val));
 980         char *point = &val[seen_point];
 981         size_t fraclen = len-seen_point-1;
 982         memmove(point, point+1, fraclen+1);
 983
 984         lit = rb_rational_new(integer_value(val, base), rb_int_positive_pow(10, fraclen));
 985     }
 986     else {
 987         lit = rb_rational_raw1(integer_value(val, base));
 988     }
 989
 990     free(val);
 991
 992     return lit;
 993 }
 994
 995 VALUE
 996 rb_node_integer_literal_val(const NODE *n)
 997 {
 998     const rb_node_integer_t *node = RNODE_INTEGER(n);
 999     VALUE val = integer_value(node->val, node->base);
1000     if (node->minus) {
1001         val = negative_numeric(val);
1002     }
1003     return val;
1004 }
1005
1006 VALUE
1007 rb_node_float_literal_val(const NODE *n)
1008 {
1009     const rb_node_float_t *node = RNODE_FLOAT(n);
1010     double d = strtod(node->val, 0);
1011     if (node->minus) {
1012         d = -d;
1013     }
1014     VALUE val = DBL2NUM(d);
1015     return val;
1016 }
1017
1018 VALUE
1019 rb_node_rational_literal_val(const NODE *n)
1020 {
1021     VALUE lit;
1022     const rb_node_rational_t *node = RNODE_RATIONAL(n);
1023
1024     lit = rational_value(node->val, node->base, node->seen_point);
1025
1026     if (node->minus) {
1027         lit = negative_numeric(lit);
1028     }
1029
1030     return lit;
1031 }
1032
1033 VALUE
1034 rb_node_imaginary_literal_val(const NODE *n)
1035 {
1036     VALUE lit;
1037     const rb_node_imaginary_t *node = RNODE_IMAGINARY(n);
1038
1039     enum rb_numeric_type type = node->type;
1040
1041     switch (type) {
1042       case integer_literal:
1043         lit = integer_value(node->val, node->base);
1044         break;
1045       case float_literal:{
1046         double d = strtod(node->val, 0);
1047         lit = DBL2NUM(d);
1048         break;
1049       }
1050       case rational_literal:
1051         lit = rational_value(node->val, node->base, node->seen_point);
1052         break;
1053       default:
1054         rb_bug("unreachable");
1055     }
1056
1057     lit = rb_complex_raw(INT2FIX(0), lit);
1058
1059     if (node->minus) {
1060         lit = negative_numeric(lit);
1061     }
1062     return lit;
1063 }
1064
1065 VALUE
1066 rb_node_str_string_val(const NODE *node)
1067 {
1068     rb_parser_string_t *str = RNODE_STR(node)->string;
1069     return rb_str_new_parser_string(str);
1070 }
1071
1072 VALUE
1073 rb_node_sym_string_val(const NODE *node)
1074 {
1075     rb_parser_string_t *str = RNODE_SYM(node)->string;
1076     return ID2SYM(rb_intern3(str->ptr, str->len, str->enc));
1077 }
1078
1079 VALUE
1080 rb_node_dstr_string_val(const NODE *node)
1081 {
1082     rb_parser_string_t *str = RNODE_DSTR(node)->string;
1083     return str ? rb_str_new_parser_string(str) : Qnil;
1084 }
1085
1086 VALUE
1087 rb_node_dregx_string_val(const NODE *node)
1088 {
1089     rb_parser_string_t *str = RNODE_DREGX(node)->string;
1090     return rb_str_new_parser_string(str);
1091 }
1092
1093 VALUE
1094 rb_node_regx_string_val(const NODE *node)
1095 {
1096     rb_node_regx_t *node_reg = RNODE_REGX(node);
1097     rb_parser_string_t *string = node_reg->string;
1098     VALUE str = rb_enc_str_new(string->ptr, string->len, string->enc);
1099
1100     return rb_reg_compile(str, node_reg->options, NULL, 0);
1101 }
1102
1103 VALUE
1104 rb_node_line_lineno_val(const NODE *node)
1105 {
1106     return INT2FIX(node->nd_loc.beg_pos.lineno);
1107 }
1108
1109 VALUE
1110 rb_node_file_path_val(const NODE *node)
1111 {
1112     return rb_str_new_parser_string(RNODE_FILE(node)->path);
1113 }
1114
1115 VALUE
1116 rb_node_encoding_val(const NODE *node)
1117 {
1118     return rb_enc_from_encoding(RNODE_ENCODING(node)->enc);
1119 }
1120
1121 static void
1122 parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *lines)
1123 {
1124     VALUE hash, script_lines;
1125     ID script_lines_id;
1126     if (NIL_P(path) || !lines) return;
1127     CONST_ID(script_lines_id, "SCRIPT_LINES__");
1128     if (!rb_const_defined_at(rb_cObject, script_lines_id)) return;
1129     hash = rb_const_get_at(rb_cObject, script_lines_id);
1130     if (!RB_TYPE_P(hash, T_HASH)) return;
1131     if (rb_hash_lookup(hash, path) == Qnil) return;
1132     script_lines = rb_parser_build_script_lines_from(lines);
1133     rb_hash_aset(hash, path, script_lines);
1134 }
1135
1136 VALUE
1137 rb_ruby_ast_new(const NODE *const root)
1138 {
1139     rb_ast_t *ast;
1140     VALUE ast_value = TypedData_Make_Struct(0, rb_ast_t, &ast_data_type, ast);
1141 #ifdef UNIVERSAL_PARSER
1142     ast->config = &rb_global_parser_config;
1143 #endif
1144     ast->body = (rb_ast_body_t){
1145         .root = root,
1146         .frozen_string_literal = -1,
1147         .coverage_enabled = -1,
1148         .script_lines = NULL,
1149         .line_count = 0,
1150     };
1151     return ast_value;
1152 }
1153
1154 rb_ast_t *
1155 rb_ruby_ast_data_get(VALUE ast_value)
1156 {
1157     rb_ast_t *ast;
1158     if (NIL_P(ast_value)) return NULL;
1159     TypedData_Get_Struct(ast_value, rb_ast_t, &ast_data_type, ast);
1160     return ast;
1161 }