ruby_parser.c

   1 /* This is a wrapper for parse.y */
   2
   3 #include "internal/parse.h"
   4 #include "internal/re.h"
   5 #include "internal/ruby_parser.h"
   6
   7 #include "node.h"
   8 #include "rubyparser.h"
   9 #include "internal/error.h"
  10
  11 #ifdef UNIVERSAL_PARSER
  12
  13 #include "internal.h"
  14 #include "internal/array.h"
  15 #include "internal/bignum.h"
  16 #include "internal/compile.h"
  17 #include "internal/complex.h"
  18 #include "internal/encoding.h"
  19 #include "internal/gc.h"
  20 #include "internal/hash.h"
  21 #include "internal/io.h"
  22 #include "internal/rational.h"
  23 #include "internal/re.h"
  24 #include "internal/string.h"
  25 #include "internal/symbol.h"
  26 #include "internal/thread.h"
  27
  28 #include "ruby/ractor.h"
  29 #include "ruby/ruby.h"
  30 #include "ruby/util.h"
  31 #include "internal.h"
  32 #include "vm_core.h"
  33 #include "symbol.h"
  34
  35 static int
  36 is_ascii_string2(VALUE str)
  37 {
  38     return is_ascii_string(str);
  39 }
  40
  41 RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 6, 0)
  42 static VALUE
  43 syntax_error_append(VALUE exc, VALUE file, int line, int column,
  44                        void *enc, const char *fmt, va_list args)
  45 {
  46     return rb_syntax_error_append(exc, file, line, column, (rb_encoding *)enc, fmt, args);
  47 }
  48
  49 static int
  50 local_defined(ID id, const void *p)
  51 {
  52     return rb_local_defined(id, (const rb_iseq_t *)p);
  53 }
  54
  55 static int
  56 dvar_defined(ID id, const void *p)
  57 {
  58     return rb_dvar_defined(id, (const rb_iseq_t *)p);
  59 }
  60
  61 static int
  62 is_usascii_enc(void *enc)
  63 {
  64     return rb_is_usascii_enc((rb_encoding *)enc);
  65 }
  66
  67 static int
  68 is_local_id2(ID id)
  69 {
  70     return is_local_id(id);
  71 }
  72
  73 static int
  74 is_attrset_id2(ID id)
  75 {
  76     return is_attrset_id(id);
  77 }
  78
  79 static int
  80 is_notop_id2(ID id)
  81 {
  82     return is_notop_id(id);
  83 }
  84
  85 static VALUE
  86 enc_str_new(const char *ptr, long len, void *enc)
  87 {
  88     return rb_enc_str_new(ptr, len, (rb_encoding *)enc);
  89 }
  90
  91 static int
  92 enc_isalnum(OnigCodePoint c, void *enc)
  93 {
  94     return rb_enc_isalnum(c, (rb_encoding *)enc);
  95 }
  96
  97 static int
  98 enc_precise_mbclen(const char *p, const char *e, void *enc)
  99 {
 100     return rb_enc_precise_mbclen(p, e, (rb_encoding *)enc);
 101 }
 102
 103 static int
 104 mbclen_charfound_p(int len)
 105 {
 106     return MBCLEN_CHARFOUND_P(len);
 107 }
 108
 109 static int
 110 mbclen_charfound_len(int len)
 111 {
 112     return MBCLEN_CHARFOUND_LEN(len);
 113 }
 114
 115 static const char *
 116 enc_name(void *enc)
 117 {
 118     return rb_enc_name((rb_encoding *)enc);
 119 }
 120
 121 static char *
 122 enc_prev_char(const char *s, const char *p, const char *e, void *enc)
 123 {
 124     return rb_enc_prev_char(s, p, e, (rb_encoding *)enc);
 125 }
 126
 127 static void *
 128 enc_get(VALUE obj)
 129 {
 130     return (void *)rb_enc_get(obj);
 131 }
 132
 133 static int
 134 enc_asciicompat(void *enc)
 135 {
 136     return rb_enc_asciicompat((rb_encoding *)enc);
 137 }
 138
 139 static void *
 140 utf8_encoding(void)
 141 {
 142     return (void *)rb_utf8_encoding();
 143 }
 144
 145 static VALUE
 146 enc_associate(VALUE obj, void *enc)
 147 {
 148     return rb_enc_associate(obj, (rb_encoding *)enc);
 149 }
 150
 151 static void *
 152 ascii8bit_encoding(void)
 153 {
 154     return (void *)rb_ascii8bit_encoding();
 155 }
 156
 157 static int
 158 enc_codelen(int c, void *enc)
 159 {
 160     return rb_enc_codelen(c, (rb_encoding *)enc);
 161 }
 162
 163 static int
 164 enc_mbcput(unsigned int c, void *buf, void *enc)
 165 {
 166     return rb_enc_mbcput(c, buf, (rb_encoding *)enc);
 167 }
 168
 169 static void *
 170 enc_from_index(int idx)
 171 {
 172     return (void *)rb_enc_from_index(idx);
 173 }
 174
 175 static int
 176 enc_isspace(OnigCodePoint c, void *enc)
 177 {
 178     return rb_enc_isspace(c, (rb_encoding *)enc);
 179 }
 180
 181 static ID
 182 intern3(const char *name, long len, void *enc)
 183 {
 184     return rb_intern3(name, len, (rb_encoding *)enc);
 185 }
 186
 187 static void *
 188 enc_compatible(VALUE str1, VALUE str2)
 189 {
 190     return (void *)rb_enc_compatible(str1, str2);
 191 }
 192
 193 static VALUE
 194 enc_from_encoding(void *enc)
 195 {
 196     return rb_enc_from_encoding((rb_encoding *)enc);
 197 }
 198
 199 static int
 200 encoding_is_ascii8bit(VALUE obj)
 201 {
 202     return ENCODING_IS_ASCII8BIT(obj);
 203 }
 204
 205 static void *
 206 usascii_encoding(void)
 207 {
 208     return (void *)rb_usascii_encoding();
 209 }
 210
 211 static int
 212 enc_symname_type(const char *name, long len, void *enc, unsigned int allowed_attrset)
 213 {
 214     return rb_enc_symname_type(name, len, (rb_encoding *)enc, allowed_attrset);
 215 }
 216
 217 typedef struct {
 218     struct parser_params *parser;
 219     rb_encoding *enc;
 220     NODE *succ_block;
 221     const rb_code_location_t *loc;
 222 } reg_named_capture_assign_t;
 223
 224 static int
 225 reg_named_capture_assign_iter(const OnigUChar *name, const OnigUChar *name_end,
 226           int back_num, int *back_refs, OnigRegex regex, void *arg0)
 227 {
 228     reg_named_capture_assign_t *arg = (reg_named_capture_assign_t*)arg0;
 229     struct parser_params* p = arg->parser;
 230     rb_encoding *enc = arg->enc;
 231     const rb_code_location_t *loc = arg->loc;
 232     long len = name_end - name;
 233     const char *s = (const char *)name;
 234
 235     return rb_reg_named_capture_assign_iter_impl(p, s, len, (void *)enc, &arg->succ_block, loc);
 236 }
 237
 238 static NODE *
 239 reg_named_capture_assign(struct parser_params* p, VALUE regexp, const rb_code_location_t *loc)
 240 {
 241     reg_named_capture_assign_t arg;
 242
 243     arg.parser = p;
 244     arg.enc = rb_enc_get(regexp);
 245     arg.succ_block = 0;
 246     arg.loc = loc;
 247     onig_foreach_name(RREGEXP_PTR(regexp), reg_named_capture_assign_iter, &arg);
 248
 249     if (!arg.succ_block) return 0;
 250     return RNODE_BLOCK(arg.succ_block)->nd_next;
 251 }
 252
 253 static int
 254 rtest(VALUE obj)
 255 {
 256     return (int)RB_TEST(obj);
 257 }
 258
 259 static int
 260 nil_p(VALUE obj)
 261 {
 262     return (int)NIL_P(obj);
 263 }
 264
 265 static VALUE
 266 syntax_error_new(void)
 267 {
 268     return rb_class_new_instance(0, 0, rb_eSyntaxError);
 269 }
 270
 271 static void *
 272 memmove2(void *dest, const void *src, size_t t, size_t n)
 273 {
 274     return memmove(dest, src, rbimpl_size_mul_or_raise(t, n));
 275 }
 276
 277 static void *
 278 nonempty_memcpy(void *dest, const void *src, size_t t, size_t n)
 279 {
 280     return ruby_nonempty_memcpy(dest, src, rbimpl_size_mul_or_raise(t, n));
 281 }
 282
 283 static VALUE
 284 ruby_verbose2(void)
 285 {
 286     return ruby_verbose;
 287 }
 288
 289 static int *
 290 rb_errno_ptr2(void)
 291 {
 292     return rb_errno_ptr();
 293 }
 294
 295 static void *
 296 zalloc(size_t elemsiz)
 297 {
 298     return ruby_xcalloc(1, elemsiz);
 299 }
 300
 301 static void
 302 gc_guard(VALUE obj)
 303 {
 304     RB_GC_GUARD(obj);
 305 }
 306
 307 static VALUE
 308 arg_error(void)
 309 {
 310     return rb_eArgError;
 311 }
 312
 313 static rb_ast_t *
 314 ast_new(VALUE nb)
 315 {
 316     return IMEMO_NEW(rb_ast_t, imemo_ast, nb);
 317 }
 318
 319 static VALUE
 320 static_id2sym(ID id)
 321 {
 322     return (((VALUE)(id)<<RUBY_SPECIAL_SHIFT)|SYMBOL_FLAG);
 323 }
 324
 325 static long
 326 str_coderange_scan_restartable(const char *s, const char *e, void *enc, int *cr)
 327 {
 328     return rb_str_coderange_scan_restartable(s, e, (rb_encoding *)enc, cr);
 329 }
 330
 331 static int
 332 enc_mbminlen(void *enc)
 333 {
 334     return rb_enc_mbminlen((rb_encoding *)enc);
 335 }
 336
 337 static bool
 338 enc_isascii(OnigCodePoint c, void *enc)
 339 {
 340     return rb_enc_isascii(c, (rb_encoding *)enc);
 341 }
 342
 343 static OnigCodePoint
 344 enc_mbc_to_codepoint(const char *p, const char *e, void *enc)
 345 {
 346     const OnigUChar *up = RBIMPL_CAST((const OnigUChar *)p);
 347     const OnigUChar *ue = RBIMPL_CAST((const OnigUChar *)e);
 348
 349     return ONIGENC_MBC_TO_CODE((rb_encoding *)enc, up, ue);
 350 }
 351
 352 VALUE rb_io_gets_internal(VALUE io);
 353 extern VALUE rb_eArgError;
 354
 355 static const rb_parser_config_t rb_global_parser_config = {
 356     .malloc = ruby_xmalloc,
 357     .calloc = ruby_xcalloc,
 358     .realloc = ruby_xrealloc,
 359     .free = ruby_xfree,
 360     .alloc_n = ruby_xmalloc2,
 361     .alloc = ruby_xmalloc,
 362     .realloc_n = ruby_xrealloc2,
 363     .zalloc = zalloc,
 364     .rb_memmove = memmove2,
 365     .nonempty_memcpy = nonempty_memcpy,
 366     .xmalloc_mul_add = rb_xmalloc_mul_add,
 367
 368     .ast_new = ast_new,
 369
 370     .compile_callback = rb_suppress_tracing,
 371     .reg_named_capture_assign = reg_named_capture_assign,
 372
 373     .attr_get = rb_attr_get,
 374
 375     .ary_new = rb_ary_new,
 376     .ary_push = rb_ary_push,
 377     .ary_new_from_args = rb_ary_new_from_args,
 378     .ary_unshift = rb_ary_unshift,
 379
 380     .make_temporary_id = rb_make_temporary_id,
 381     .is_local_id = is_local_id2,
 382     .is_attrset_id = is_attrset_id2,
 383     .is_global_name_punct = is_global_name_punct,
 384     .id_type = id_type,
 385     .id_attrset = rb_id_attrset,
 386     .intern = rb_intern,
 387     .intern2 = rb_intern2,
 388     .intern3 = intern3,
 389     .intern_str = rb_intern_str,
 390     .is_notop_id = is_notop_id2,
 391     .enc_symname_type = enc_symname_type,
 392     .id2name = rb_id2name,
 393     .id2str = rb_id2str,
 394     .id2sym = rb_id2sym,
 395     .sym2id = rb_sym2id,
 396
 397     .str_catf = rb_str_catf,
 398     .str_cat_cstr = rb_str_cat_cstr,
 399     .str_subseq = rb_str_subseq,
 400     .str_new_frozen = rb_str_new_frozen,
 401     .str_modify = rb_str_modify,
 402     .str_set_len = rb_str_set_len,
 403     .str_cat = rb_str_cat,
 404     .str_resize = rb_str_resize,
 405     .str_new = rb_str_new,
 406     .str_new_cstr = rb_str_new_cstr,
 407     .str_to_interned_str = rb_str_to_interned_str,
 408     .is_ascii_string = is_ascii_string2,
 409     .enc_str_new = enc_str_new,
 410     .str_vcatf = rb_str_vcatf,
 411     .string_value_cstr = rb_string_value_cstr,
 412     .rb_sprintf = rb_sprintf,
 413     .rstring_ptr = RSTRING_PTR,
 414     .rstring_end = RSTRING_END,
 415     .rstring_len = RSTRING_LEN,
 416     .filesystem_str_new_cstr = rb_filesystem_str_new_cstr,
 417     .obj_as_string = rb_obj_as_string,
 418
 419     .int2num = rb_int2num_inline,
 420
 421     .stderr_tty_p = rb_stderr_tty_p,
 422     .write_error_str = rb_write_error_str,
 423     .io_write = rb_io_write,
 424     .io_flush = rb_io_flush,
 425     .io_puts = rb_io_puts,
 426     .io_gets_internal = rb_io_gets_internal,
 427
 428     .debug_output_stdout = rb_ractor_stdout,
 429     .debug_output_stderr = rb_ractor_stderr,
 430
 431     .is_usascii_enc = is_usascii_enc,
 432     .enc_isalnum = enc_isalnum,
 433     .enc_precise_mbclen = enc_precise_mbclen,
 434     .mbclen_charfound_p = mbclen_charfound_p,
 435     .mbclen_charfound_len = mbclen_charfound_len,
 436     .enc_name = enc_name,
 437     .enc_prev_char = enc_prev_char,
 438     .enc_get = enc_get,
 439     .enc_asciicompat = enc_asciicompat,
 440     .utf8_encoding = utf8_encoding,
 441     .enc_associate = enc_associate,
 442     .ascii8bit_encoding = ascii8bit_encoding,
 443     .enc_codelen = enc_codelen,
 444     .enc_mbcput = enc_mbcput,
 445     .enc_find_index = rb_enc_find_index,
 446     .enc_from_index = enc_from_index,
 447     .enc_isspace = enc_isspace,
 448     .enc_coderange_7bit = ENC_CODERANGE_7BIT,
 449     .enc_coderange_unknown = ENC_CODERANGE_UNKNOWN,
 450     .enc_compatible = enc_compatible,
 451     .enc_from_encoding = enc_from_encoding,
 452     .encoding_is_ascii8bit = encoding_is_ascii8bit,
 453     .usascii_encoding = usascii_encoding,
 454     .enc_coderange_broken = ENC_CODERANGE_BROKEN,
 455     .enc_mbminlen = enc_mbminlen,
 456     .enc_isascii = enc_isascii,
 457     .enc_mbc_to_codepoint = enc_mbc_to_codepoint,
 458
 459     .local_defined = local_defined,
 460     .dvar_defined = dvar_defined,
 461
 462     .syntax_error_append = syntax_error_append,
 463     .raise = rb_raise,
 464     .syntax_error_new = syntax_error_new,
 465
 466     .errinfo = rb_errinfo,
 467     .set_errinfo = rb_set_errinfo,
 468     .exc_raise = rb_exc_raise,
 469     .make_exception = rb_make_exception,
 470
 471     .sized_xfree = ruby_sized_xfree,
 472     .sized_realloc_n = ruby_sized_realloc_n,
 473     .gc_guard = gc_guard,
 474     .gc_mark = rb_gc_mark,
 475
 476     .reg_compile = rb_reg_compile,
 477     .reg_check_preprocess = rb_reg_check_preprocess,
 478     .memcicmp = rb_memcicmp,
 479
 480     .compile_warn = rb_compile_warn,
 481     .compile_warning = rb_compile_warning,
 482     .bug = rb_bug,
 483     .fatal = rb_fatal,
 484     .verbose = ruby_verbose2,
 485     .errno_ptr = rb_errno_ptr2,
 486
 487     .make_backtrace = rb_make_backtrace,
 488
 489     .scan_hex = ruby_scan_hex,
 490     .scan_oct = ruby_scan_oct,
 491     .scan_digits = ruby_scan_digits,
 492     .strtod = ruby_strtod,
 493
 494     .rtest = rtest,
 495     .nil_p = nil_p,
 496     .qnil = Qnil,
 497     .qfalse = Qfalse,
 498     .eArgError = arg_error,
 499     .long2int = rb_long2int,
 500
 501     /* For Ripper */
 502     .static_id2sym = static_id2sym,
 503     .str_coderange_scan_restartable = str_coderange_scan_restartable,
 504 };
 505 #endif
 506
 507 enum lex_type {
 508     lex_type_str,
 509     lex_type_io,
 510     lex_type_array,
 511     lex_type_generic,
 512 };
 513
 514 struct ruby_parser {
 515     rb_parser_t *parser_params;
 516     enum lex_type type;
 517     union {
 518         struct lex_pointer_string lex_str;
 519         struct {
 520             VALUE file;
 521         } lex_io;
 522         struct {
 523             VALUE ary;
 524         } lex_array;
 525     } data;
 526 };
 527
 528 static void
 529 parser_mark(void *ptr)
 530 {
 531     struct ruby_parser *parser = (struct ruby_parser*)ptr;
 532     rb_ruby_parser_mark(parser->parser_params);
 533
 534     switch (parser->type) {
 535       case lex_type_str:
 536         rb_gc_mark(parser->data.lex_str.str);
 537         break;
 538       case lex_type_io:
 539         rb_gc_mark(parser->data.lex_io.file);
 540         break;
 541       case lex_type_array:
 542         rb_gc_mark(parser->data.lex_array.ary);
 543         break;
 544       case lex_type_generic:
 545         /* noop. Caller of rb_parser_compile_generic should mark the objects. */
 546         break;
 547     }
 548 }
 549
 550 static void
 551 parser_free(void *ptr)
 552 {
 553     struct ruby_parser *parser = (struct ruby_parser*)ptr;
 554     rb_ruby_parser_free(parser->parser_params);
 555 }
 556
 557 static size_t
 558 parser_memsize(const void *ptr)
 559 {
 560     struct ruby_parser *parser = (struct ruby_parser*)ptr;
 561     return rb_ruby_parser_memsize(parser->parser_params);
 562 }
 563
 564 static const rb_data_type_t ruby_parser_data_type = {
 565     "parser",
 566     {
 567         parser_mark,
 568         parser_free,
 569         parser_memsize,
 570     },
 571     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
 572 };
 573
 574 #ifdef UNIVERSAL_PARSER
 575 const rb_parser_config_t *
 576 rb_ruby_parser_config(void)
 577 {
 578     return &rb_global_parser_config;
 579 }
 580
 581 rb_parser_t *
 582 rb_parser_params_new(void)
 583 {
 584     return rb_ruby_parser_new(&rb_global_parser_config);
 585 }
 586 #else
 587 rb_parser_t *
 588 rb_parser_params_new(void)
 589 {
 590     return rb_ruby_parser_new();
 591 }
 592 #endif /* UNIVERSAL_PARSER */
 593
 594 VALUE
 595 rb_parser_new(void)
 596 {
 597     struct ruby_parser *parser;
 598     rb_parser_t *parser_params;
 599
 600     /*
 601      * Create parser_params ahead of vparser because
 602      * rb_ruby_parser_new can run GC so if create vparser
 603      * first, parser_mark tries to mark not initialized parser_params.
 604      */
 605     parser_params = rb_parser_params_new();
 606     VALUE vparser = TypedData_Make_Struct(0, struct ruby_parser,
 607                                          &ruby_parser_data_type, parser);
 608     parser->parser_params = parser_params;
 609
 610     return vparser;
 611 }
 612
 613 void
 614 rb_parser_set_options(VALUE vparser, int print, int loop, int chomp, int split)
 615 {
 616     struct ruby_parser *parser;
 617
 618     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 619     rb_ruby_parser_set_options(parser->parser_params, print, loop, chomp, split);
 620 }
 621
 622 VALUE
 623 rb_parser_set_context(VALUE vparser, const struct rb_iseq_struct *base, int main)
 624 {
 625     struct ruby_parser *parser;
 626
 627     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 628     rb_ruby_parser_set_context(parser->parser_params, base, main);
 629     return vparser;
 630 }
 631
 632 void
 633 rb_parser_set_script_lines(VALUE vparser)
 634 {
 635     struct ruby_parser *parser;
 636
 637     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 638     rb_ruby_parser_set_script_lines(parser->parser_params);
 639 }
 640
 641 void
 642 rb_parser_error_tolerant(VALUE vparser)
 643 {
 644     struct ruby_parser *parser;
 645
 646     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 647     rb_ruby_parser_error_tolerant(parser->parser_params);
 648 }
 649
 650 void
 651 rb_parser_keep_tokens(VALUE vparser)
 652 {
 653     struct ruby_parser *parser;
 654
 655     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 656     rb_ruby_parser_keep_tokens(parser->parser_params);
 657 }
 658
 659 VALUE
 660 rb_parser_lex_get_str(struct lex_pointer_string *ptr_str)
 661 {
 662     char *beg, *end, *start;
 663     long len;
 664     VALUE s = ptr_str->str;
 665
 666     beg = RSTRING_PTR(s);
 667     len = RSTRING_LEN(s);
 668     start = beg;
 669     if (ptr_str->ptr) {
 670         if (len == ptr_str->ptr) return Qnil;
 671         beg += ptr_str->ptr;
 672         len -= ptr_str->ptr;
 673     }
 674     end = memchr(beg, '\n', len);
 675     if (end) len = ++end - beg;
 676     ptr_str->ptr += len;
 677     return rb_str_subseq(s, beg - start, len);
 678 }
 679
 680 static VALUE
 681 lex_get_str(struct parser_params *p, rb_parser_input_data input, int line_count)
 682 {
 683     return rb_parser_lex_get_str((struct lex_pointer_string *)input);
 684 }
 685
 686 static rb_ast_t*
 687 parser_compile_string0(struct ruby_parser *parser, VALUE fname, VALUE s, int line)
 688 {
 689     VALUE str = rb_str_new_frozen(s);
 690
 691     parser->type = lex_type_str;
 692     parser->data.lex_str.str = str;
 693     parser->data.lex_str.ptr = 0;
 694
 695     return rb_parser_compile(parser->parser_params, lex_get_str, fname, (rb_parser_input_data)&parser->data, line);
 696 }
 697
 698 static rb_encoding *
 699 must_be_ascii_compatible(VALUE s)
 700 {
 701     rb_encoding *enc = rb_enc_get(s);
 702     if (!rb_enc_asciicompat(enc)) {
 703         rb_raise(rb_eArgError, "invalid source encoding");
 704     }
 705     return enc;
 706 }
 707
 708 static rb_ast_t*
 709 parser_compile_string_path(struct ruby_parser *parser, VALUE f, VALUE s, int line)
 710 {
 711     must_be_ascii_compatible(s);
 712     return parser_compile_string0(parser, f, s, line);
 713 }
 714
 715 static rb_ast_t*
 716 parser_compile_string(struct ruby_parser *parser, const char *f, VALUE s, int line)
 717 {
 718     return parser_compile_string_path(parser, rb_filesystem_str_new_cstr(f), s, line);
 719 }
 720
 721 VALUE rb_io_gets_internal(VALUE io);
 722
 723 static VALUE
 724 lex_io_gets(struct parser_params *p, rb_parser_input_data input, int line_count)
 725 {
 726     VALUE io = (VALUE)input;
 727
 728     return rb_io_gets_internal(io);
 729 }
 730
 731 static VALUE
 732 lex_gets_array(struct parser_params *p, rb_parser_input_data data, int index)
 733 {
 734     VALUE array = (VALUE)data;
 735     VALUE str = rb_ary_entry(array, index);
 736     if (!NIL_P(str)) {
 737         StringValue(str);
 738         if (!rb_enc_asciicompat(rb_enc_get(str))) {
 739             rb_raise(rb_eArgError, "invalid source encoding");
 740         }
 741     }
 742     return str;
 743 }
 744
 745 static rb_ast_t*
 746 parser_compile_file_path(struct ruby_parser *parser, VALUE fname, VALUE file, int start)
 747 {
 748     parser->type = lex_type_io;
 749     parser->data.lex_io.file = file;
 750
 751     return rb_parser_compile(parser->parser_params, lex_io_gets, fname, (rb_parser_input_data)file, start);
 752 }
 753
 754 static rb_ast_t*
 755 parser_compile_array(struct ruby_parser *parser, VALUE fname, VALUE array, int start)
 756 {
 757     parser->type = lex_type_array;
 758     parser->data.lex_array.ary = array;
 759
 760     return rb_parser_compile(parser->parser_params, lex_gets_array, fname, (rb_parser_input_data)array, start);
 761 }
 762
 763 static rb_ast_t*
 764 parser_compile_generic(struct ruby_parser *parser, rb_parser_lex_gets_func *lex_gets, VALUE fname, VALUE input, int start)
 765 {
 766     parser->type = lex_type_generic;
 767
 768     return rb_parser_compile(parser->parser_params, lex_gets, fname, (rb_parser_input_data)input, start);
 769 }
 770
 771 rb_ast_t*
 772 rb_parser_compile_file_path(VALUE vparser, VALUE fname, VALUE file, int start)
 773 {
 774     struct ruby_parser *parser;
 775     rb_ast_t *ast;
 776
 777     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 778     ast = parser_compile_file_path(parser, fname, file, start);
 779     RB_GC_GUARD(vparser);
 780
 781     return ast;
 782 }
 783
 784 rb_ast_t*
 785 rb_parser_compile_array(VALUE vparser, VALUE fname, VALUE array, int start)
 786 {
 787     struct ruby_parser *parser;
 788     rb_ast_t *ast;
 789
 790     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 791     ast = parser_compile_array(parser, fname, array, start);
 792     RB_GC_GUARD(vparser);
 793
 794     return ast;
 795 }
 796
 797 rb_ast_t*
 798 rb_parser_compile_generic(VALUE vparser, rb_parser_lex_gets_func *lex_gets, VALUE fname, VALUE input, int start)
 799 {
 800     struct ruby_parser *parser;
 801     rb_ast_t *ast;
 802
 803     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 804     ast = parser_compile_generic(parser, lex_gets, fname, input, start);
 805     RB_GC_GUARD(vparser);
 806
 807     return ast;
 808 }
 809
 810 rb_ast_t*
 811 rb_parser_compile_string(VALUE vparser, const char *f, VALUE s, int line)
 812 {
 813     struct ruby_parser *parser;
 814     rb_ast_t *ast;
 815
 816     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 817     ast = parser_compile_string(parser, f, s, line);
 818     RB_GC_GUARD(vparser);
 819
 820     return ast;
 821 }
 822
 823 rb_ast_t*
 824 rb_parser_compile_string_path(VALUE vparser, VALUE f, VALUE s, int line)
 825 {
 826     struct ruby_parser *parser;
 827     rb_ast_t *ast;
 828
 829     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 830     ast = parser_compile_string_path(parser, f, s, line);
 831     RB_GC_GUARD(vparser);
 832
 833     return ast;
 834 }
 835
 836 VALUE
 837 rb_parser_encoding(VALUE vparser)
 838 {
 839     struct ruby_parser *parser;
 840
 841     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 842     return rb_ruby_parser_encoding(parser->parser_params);
 843 }
 844
 845 VALUE
 846 rb_parser_end_seen_p(VALUE vparser)
 847 {
 848     struct ruby_parser *parser;
 849
 850     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 851     return RBOOL(rb_ruby_parser_end_seen_p(parser->parser_params));
 852 }
 853
 854 VALUE
 855 rb_parser_set_yydebug(VALUE vparser, VALUE flag)
 856 {
 857     struct ruby_parser *parser;
 858
 859     TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 860     rb_ruby_parser_set_yydebug(parser->parser_params, RTEST(flag));
 861     return flag;
 862 }
 863
 864 void
 865 rb_set_script_lines_for(VALUE vparser, VALUE path)
 866 {
 867     struct ruby_parser *parser;
 868     VALUE hash;
 869     ID script_lines;
 870     CONST_ID(script_lines, "SCRIPT_LINES__");
 871     if (!rb_const_defined_at(rb_cObject, script_lines)) return;
 872     hash = rb_const_get_at(rb_cObject, script_lines);
 873     if (RB_TYPE_P(hash, T_HASH)) {
 874         rb_hash_aset(hash, path, Qtrue);
 875         TypedData_Get_Struct(vparser, struct ruby_parser, &ruby_parser_data_type, parser);
 876         rb_ruby_parser_set_script_lines(parser->parser_params);
 877     }
 878 }
 879
 880 VALUE
 881 rb_parser_build_script_lines_from(rb_parser_ary_t *lines)
 882 {
 883     int i;
 884     if (lines->data_type != PARSER_ARY_DATA_SCRIPT_LINE) {
 885         rb_bug("unexpected rb_parser_ary_data_type (%d) for script lines", lines->data_type);
 886     }
 887     VALUE script_lines = rb_ary_new_capa(lines->len);
 888     for (i = 0; i < lines->len; i++) {
 889         rb_parser_string_t *str = (rb_parser_string_t *)lines->data[i];
 890         rb_ary_push(script_lines, rb_enc_str_new(str->ptr, str->len, str->enc));
 891     }
 892     return script_lines;
 893 }
 894
 895 VALUE
 896 rb_str_new_parser_string(rb_parser_string_t *str)
 897 {
 898     VALUE string = rb_enc_interned_str(str->ptr, str->len, str->enc);
 899     rb_enc_str_coderange(string);
 900     return string;
 901 }
 902
 903 VALUE
 904 rb_str_new_mutable_parser_string(rb_parser_string_t *str)
 905 {
 906     return rb_enc_str_new(str->ptr, str->len, str->enc);
 907 }
 908
 909 static VALUE
 910 negative_numeric(VALUE val)
 911 {
 912     if (FIXNUM_P(val)) {
 913         return LONG2FIX(-FIX2LONG(val));
 914     }
 915     if (SPECIAL_CONST_P(val)) {
 916 #if USE_FLONUM
 917         if (FLONUM_P(val)) {
 918             return DBL2NUM(-RFLOAT_VALUE(val));
 919         }
 920 #endif
 921         goto unknown;
 922     }
 923     switch (BUILTIN_TYPE(val)) {
 924       case T_BIGNUM:
 925         BIGNUM_NEGATE(val);
 926         val = rb_big_norm(val);
 927         break;
 928       case T_RATIONAL:
 929         RATIONAL_SET_NUM(val, negative_numeric(RRATIONAL(val)->num));
 930         break;
 931       case T_COMPLEX:
 932         RCOMPLEX_SET_REAL(val, negative_numeric(RCOMPLEX(val)->real));
 933         RCOMPLEX_SET_IMAG(val, negative_numeric(RCOMPLEX(val)->imag));
 934         break;
 935       case T_FLOAT:
 936         val = DBL2NUM(-RFLOAT_VALUE(val));
 937         break;
 938       unknown:
 939       default:
 940         rb_bug("unknown literal type (%s) passed to negative_numeric",
 941                rb_builtin_class_name(val));
 942         break;
 943     }
 944     return val;
 945 }
 946
 947 static VALUE
 948 integer_value(const char *val, int base)
 949 {
 950     return rb_cstr_to_inum(val, base, FALSE);
 951 }
 952
 953 static VALUE
 954 rational_value(const char *node_val, int base, int seen_point)
 955 {
 956     VALUE lit;
 957     char* val = strdup(node_val);
 958     if (seen_point > 0) {
 959         int len = (int)(strlen(val));
 960         char *point = &val[seen_point];
 961         size_t fraclen = len-seen_point-1;
 962         memmove(point, point+1, fraclen+1);
 963
 964         lit = rb_rational_new(integer_value(val, base), rb_int_positive_pow(10, fraclen));
 965     }
 966     else {
 967         lit = rb_rational_raw1(integer_value(val, base));
 968     }
 969
 970     free(val);
 971
 972     return lit;
 973 }
 974
 975 VALUE
 976 rb_node_integer_literal_val(const NODE *n)
 977 {
 978     const rb_node_integer_t *node = RNODE_INTEGER(n);
 979     VALUE val = integer_value(node->val, node->base);
 980     if (node->minus) {
 981         val = negative_numeric(val);
 982     }
 983     return val;
 984 }
 985
 986 VALUE
 987 rb_node_float_literal_val(const NODE *n)
 988 {
 989     const rb_node_float_t *node = RNODE_FLOAT(n);
 990     double d = strtod(node->val, 0);
 991     if (node->minus) {
 992         d = -d;
 993     }
 994     VALUE val = DBL2NUM(d);
 995     return val;
 996 }
 997
 998 VALUE
 999 rb_node_rational_literal_val(const NODE *n)
1000 {
1001     VALUE lit;
1002     const rb_node_rational_t *node = RNODE_RATIONAL(n);
1003
1004     lit = rational_value(node->val, node->base, node->seen_point);
1005
1006     if (node->minus) {
1007         lit = negative_numeric(lit);
1008     }
1009
1010     return lit;
1011 }
1012
1013 VALUE
1014 rb_node_imaginary_literal_val(const NODE *n)
1015 {
1016     VALUE lit;
1017     const rb_node_imaginary_t *node = RNODE_IMAGINARY(n);
1018
1019     enum rb_numeric_type type = node->type;
1020
1021     switch (type) {
1022       case integer_literal:
1023         lit = integer_value(node->val, node->base);
1024         break;
1025       case float_literal:{
1026         double d = strtod(node->val, 0);
1027         lit = DBL2NUM(d);
1028         break;
1029       }
1030       case rational_literal:
1031         lit = rational_value(node->val, node->base, node->seen_point);
1032         break;
1033       default:
1034         rb_bug("unreachable");
1035     }
1036
1037     lit = rb_complex_raw(INT2FIX(0), lit);
1038
1039     if (node->minus) {
1040         lit = negative_numeric(lit);
1041     }
1042     return lit;
1043 }
1044
1045 VALUE
1046 rb_node_str_string_val(const NODE *node)
1047 {
1048     rb_parser_string_t *str = RNODE_STR(node)->string;
1049     return rb_str_new_parser_string(str);
1050 }
1051
1052 VALUE
1053 rb_node_sym_string_val(const NODE *node)
1054 {
1055     rb_parser_string_t *str = RNODE_SYM(node)->string;
1056     return ID2SYM(rb_intern3(str->ptr, str->len, str->enc));
1057 }
1058
1059 VALUE
1060 rb_node_dstr_string_val(const NODE *node)
1061 {
1062     rb_parser_string_t *str = RNODE_DSTR(node)->string;
1063     return str ? rb_str_new_parser_string(str) : Qnil;
1064 }
1065
1066 VALUE
1067 rb_node_dregx_string_val(const NODE *node)
1068 {
1069     rb_parser_string_t *str = RNODE_DREGX(node)->string;
1070     return rb_str_new_parser_string(str);
1071 }
1072
1073 VALUE
1074 rb_node_regx_string_val(const NODE *node)
1075 {
1076     rb_node_regx_t *node_reg = RNODE_REGX(node);
1077     rb_parser_string_t *string = node_reg->string;
1078     VALUE str = rb_enc_str_new(string->ptr, string->len, string->enc);
1079
1080     return rb_reg_compile(str, node_reg->options, NULL, 0);
1081 }
1082
1083 VALUE
1084 rb_node_line_lineno_val(const NODE *node)
1085 {
1086     return INT2FIX(node->nd_loc.beg_pos.lineno);
1087 }
1088
1089 VALUE
1090 rb_node_file_path_val(const NODE *node)
1091 {
1092     return rb_str_new_parser_string(RNODE_FILE(node)->path);
1093 }
1094
1095 VALUE
1096 rb_node_encoding_val(const NODE *node)
1097 {
1098     return rb_enc_from_encoding(RNODE_ENCODING(node)->enc);
1099 }
1100
1101 void
1102 rb_parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *lines)
1103 {
1104     VALUE hash, script_lines;
1105     ID script_lines_id;
1106     if (NIL_P(path) || !lines || FIXNUM_P((VALUE)lines)) return;
1107     CONST_ID(script_lines_id, "SCRIPT_LINES__");
1108     if (!rb_const_defined_at(rb_cObject, script_lines_id)) return;
1109     hash = rb_const_get_at(rb_cObject, script_lines_id);
1110     if (!RB_TYPE_P(hash, T_HASH)) return;
1111     if (rb_hash_lookup(hash, path) == Qnil) return;
1112     script_lines = rb_parser_build_script_lines_from(lines);
1113     rb_hash_aset(hash, path, script_lines);
1114 }