[Bug #20270] Fix --parser=prism (#10970)

Co-authored-by: Takashi Kokubun <[email protected]>
author: Peter Zhu <[email protected]> 2024-06-11 19:43:32 -0400
committer: GitHub <[email protected]> 2024-06-11 16:43:32 -0700
commit: 97b1bf9ac11848c2783264d22bf7cdb7f32a21cf (patch)
tree: b8d5259e382badfdf4e857c4e12fb5d4da2f5bee
parent: d1869cfb852cf95b5a51025c016437ab46b12104 (diff)
2 files changed, 130 insertions, 127 deletions
diff --git a/ruby.c b/ruby.c
index fedb42c743..821213484e 100644
--- a/ruby.c
+++ b/ruby.c
@@ -153,8 +153,6 @@ enum feature_flag_bits {
     SEP \
     X(parsetree_with_comment) \
     SEP \
-    X(prism_parsetree) \
-    SEP \
     X(insns) \
     SEP \
     X(insns_without_opt) \
@@ -168,7 +166,7 @@ enum dump_flag_bits {
                                 DUMP_BIT(parsetree_with_comment)),
     dump_exit_bits = (DUMP_BIT(yydebug) | DUMP_BIT(syntax) |
                       DUMP_BIT(parsetree) | DUMP_BIT(parsetree_with_comment) |
-                      DUMP_BIT(prism_parsetree) | DUMP_BIT(insns) | DUMP_BIT(insns_without_opt))
+                      DUMP_BIT(insns) | DUMP_BIT(insns_without_opt))
 };
 
 static inline void
@@ -355,7 +353,7 @@ usage(const char *name, int help, int highlight, int columns)
 
     static const struct ruby_opt_message help_msg[] = {
         M("--copyright",                            "", "print the copyright"),
-        M("--dump={insns|parsetree|prism_parsetree|...}[,...]",     "",
+        M("--dump={insns|parsetree|...}[,...]",     "",
           "dump debug information. see below for available dump list"),
         M("--enable={jit|rubyopt|...}[,...]", ", --disable={jit|rubyopt|...}[,...]",
           "enable or disable features. see below for available features"),
@@ -1986,12 +1984,96 @@ env_var_truthy(const char *name)
 
 rb_pid_t rb_fork_ruby(int *status);
 
+static rb_ast_t *
+process_script(ruby_cmdline_options_t *opt)
+{
+    rb_ast_t *ast;
+    VALUE parser = rb_parser_new();
+
+    if (opt->dump & DUMP_BIT(yydebug)) {
+        rb_parser_set_yydebug(parser, Qtrue);
+    }
+
+    if (opt->dump & DUMP_BIT(error_tolerant)) {
+        rb_parser_error_tolerant(parser);
+    }
+
+    if (opt->e_script) {
+        VALUE progname = rb_progname;
+        rb_parser_set_context(parser, 0, TRUE);
+
+        ruby_opt_init(opt);
+        ruby_set_script_name(progname);
+        rb_parser_set_options(parser, opt->do_print, opt->do_loop,
+                              opt->do_line, opt->do_split);
+        ast = rb_parser_compile_string(parser, opt->script, opt->e_script, 1);
+    }
+    else {
+        VALUE f;
+        int xflag = opt->xflag;
+        f = open_load_file(opt->script_name, &xflag);
+        opt->xflag = xflag != 0;
+        rb_parser_set_context(parser, 0, f == rb_stdin);
+        ast = load_file(parser, opt->script_name, f, 1, opt);
+    }
+    if (!ast->body.root) {
+        rb_ast_dispose(ast);
+        return NULL;
+    }
+    return ast;
+}
+
+static void
+prism_script(ruby_cmdline_options_t *opt, pm_string_t *input, pm_options_t *options)
+{
+    ruby_opt_init(opt);
+
+    if (strcmp(opt->script, "-") == 0) {
+        rb_warn("Prism support for streaming code from stdin is not currently supported");
+        pm_string_constant_init(input, "", 0);
+        pm_options_filepath_set(options, "-e");
+    }
+    else if (opt->e_script) {
+        pm_string_constant_init(input, RSTRING_PTR(opt->e_script), RSTRING_LEN(opt->e_script));
+        pm_options_filepath_set(options, "-e");
+    }
+    else {
+        pm_string_mapped_init(input, RSTRING_PTR(opt->script_name));
+        pm_options_filepath_set(options, RSTRING_PTR(opt->script_name));
+    }
+}
+
+static VALUE
+prism_dump_tree(pm_string_t *input, pm_options_t *options)
+{
+    pm_parser_t parser;
+    pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
+
+    pm_node_t *node = pm_parse(&parser);
+
+    pm_buffer_t output_buffer = { 0 };
+
+    pm_prettyprint(&output_buffer, &parser, node);
+
+    VALUE tree = rb_str_new(output_buffer.value, output_buffer.length);
+
+    pm_buffer_free(&output_buffer);
+    pm_node_destroy(&parser, node);
+    pm_parser_free(&parser);
+
+    return tree;
+}
+
 static VALUE
 process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
 {
-    rb_ast_t *ast = 0;
-    VALUE parser;
-    VALUE script_name;
+    rb_ast_t *ast = NULL;
+    pm_string_t pm_input = { 0 };
+    pm_options_t pm_options = { 0 };
+
+#define dispose_result() \
+    (ast ? rb_ast_dispose(ast) : (pm_string_free(&pm_input), pm_options_free(&pm_options)))
+
     const rb_iseq_t *iseq;
     rb_encoding *enc, *lenc;
 #if UTF8_PATH
@@ -2162,13 +2244,6 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
     lenc = rb_locale_encoding();
     rb_enc_associate(rb_progname, lenc);
     rb_obj_freeze(rb_progname);
-    parser = rb_parser_new();
-    if (opt->dump & DUMP_BIT(yydebug)) {
-        rb_parser_set_yydebug(parser, Qtrue);
-    }
-    if (opt->dump & DUMP_BIT(error_tolerant)) {
-        rb_parser_error_tolerant(parser);
-    }
     if (opt->ext.enc.name != 0) {
         opt->ext.enc.index = opt_enc_index(opt->ext.enc.name);
     }
@@ -2194,7 +2269,6 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
         ienc = enc;
 #endif
     }
-    script_name = opt->script_name;
     rb_enc_associate(opt->script_name, IF_UTF8_PATH(uenc, lenc));
 #if UTF8_PATH
     if (uenc != lenc) {
@@ -2261,46 +2335,35 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
     ruby_set_argv(argc, argv);
     opt->sflag = process_sflag(opt->sflag);
 
-    if (!(*rb_ruby_prism_ptr())) {
-        if (opt->e_script) {
-            VALUE progname = rb_progname;
-            rb_encoding *eenc;
-            rb_parser_set_context(parser, 0, TRUE);
-
-            if (opt->src.enc.index >= 0) {
-                eenc = rb_enc_from_index(opt->src.enc.index);
-            }
-            else {
-                eenc = lenc;
-#if UTF8_PATH
-                if (ienc) eenc = ienc;
-#endif
-            }
+   if (opt->e_script) {
+        rb_encoding *eenc;
+        if (opt->src.enc.index >= 0) {
+            eenc = rb_enc_from_index(opt->src.enc.index);
+        }
+        else {
+            eenc = lenc;
 #if UTF8_PATH
-            if (eenc != uenc) {
-                opt->e_script = str_conv_enc(opt->e_script, uenc, eenc);
-            }
+            if (ienc) eenc = ienc;
 #endif
-            rb_enc_associate(opt->e_script, eenc);
-            ruby_opt_init(opt);
-            ruby_set_script_name(progname);
-            rb_parser_set_options(parser, opt->do_print, opt->do_loop,
-                                  opt->do_line, opt->do_split);
-            ast = rb_parser_compile_string(parser, opt->script, opt->e_script, 1);
         }
-        else {
-            VALUE f;
-            int xflag = opt->xflag;
-            f = open_load_file(script_name, &xflag);
-            opt->xflag = xflag != 0;
-            rb_parser_set_context(parser, 0, f == rb_stdin);
-            ast = load_file(parser, opt->script_name, f, 1, opt);
+#if UTF8_PATH
+        if (eenc != uenc) {
+            opt->e_script = str_conv_enc(opt->e_script, uenc, eenc);
         }
+#endif
+        rb_enc_associate(opt->e_script, eenc);
+   }
+
+    if (!(*rb_ruby_prism_ptr())) {
+        if (!(ast = process_script(opt))) return Qfalse;
+    }
+    else {
+        prism_script(opt, &pm_input, &pm_options);
     }
     ruby_set_script_name(opt->script_name);
-    if (dump & DUMP_BIT(yydebug)) {
-        dump &= ~DUMP_BIT(yydebug);
-        if (!dump) return Qtrue;
+    if ((dump & DUMP_BIT(yydebug)) && !(dump &= ~DUMP_BIT(yydebug))) {
+        dispose_result();
+        return Qtrue;
     }
 
     if (opt->ext.enc.index >= 0) {
@@ -2320,11 +2383,6 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
         rb_enc_set_default_internal(Qnil);
     rb_stdio_set_default_encoding();
 
-    if (!(*rb_ruby_prism_ptr()) && !ast->body.root) {
-        rb_ast_dispose(ast);
-        return Qfalse;
-    }
-
     opt->sflag = process_sflag(opt->sflag);
     opt->xflag = 0;
 
@@ -2341,52 +2399,20 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
         rb_define_global_function("chomp", rb_f_chomp, -1);
     }
 
-    if (dump & (DUMP_BIT(prism_parsetree))) {
-        pm_string_t input;
-        pm_options_t options = { 0 };
-
-        if (strcmp(opt->script, "-") == 0) {
-            int xflag = opt->xflag;
-            VALUE rb_source = open_load_file(opt->script_name, &xflag);
-            opt->xflag = xflag != 0;
-
-            rb_warn("Prism support for streaming code from stdin is not currently supported");
-            pm_string_constant_init(&input, RSTRING_PTR(rb_source), RSTRING_LEN(rb_source));
-            pm_options_filepath_set(&options, RSTRING_PTR(opt->script_name));
-        }
-        else if (opt->e_script) {
-            pm_string_constant_init(&input, RSTRING_PTR(opt->e_script), RSTRING_LEN(opt->e_script));
-            pm_options_filepath_set(&options, "-e");
+    if (dump & (DUMP_BIT(parsetree)|DUMP_BIT(parsetree_with_comment))) {
+        VALUE tree;
+        if (ast) {
+            int comment = dump & DUMP_BIT(parsetree_with_comment);
+            tree = rb_parser_dump_tree(ast->body.root, comment);
         }
         else {
-            pm_string_mapped_init(&input, RSTRING_PTR(opt->script_name));
-            pm_options_filepath_set(&options, RSTRING_PTR(opt->script_name));
+            tree = prism_dump_tree(&pm_input, &pm_options);
         }
-
-        pm_parser_t parser;
-        pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), &options);
-
-        pm_node_t *node = pm_parse(&parser);
-        pm_buffer_t output_buffer = { 0 };
-
-        pm_prettyprint(&output_buffer, &parser, node);
-        rb_io_write(rb_stdout, rb_str_new((const char *) output_buffer.value, output_buffer.length));
-        rb_io_flush(rb_stdout);
-
-        pm_buffer_free(&output_buffer);
-        pm_node_destroy(&parser, node);
-        pm_parser_free(&parser);
-
-        pm_string_free(&input);
-        pm_options_free(&options);
-    }
-
-    if (dump & (DUMP_BIT(parsetree)|DUMP_BIT(parsetree_with_comment))) {
-        rb_io_write(rb_stdout, rb_parser_dump_tree(ast->body.root, dump & DUMP_BIT(parsetree_with_comment)));
+        rb_io_write(rb_stdout, tree);
         rb_io_flush(rb_stdout);
         dump &= ~DUMP_BIT(parsetree)&~DUMP_BIT(parsetree_with_comment);
         if (!dump) {
-            rb_ast_dispose(ast);
+            dispose_result();
             return Qtrue;
         }
     }
@@ -2394,7 +2420,7 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
     {
         VALUE path = Qnil;
         if (!opt->e_script && strcmp(opt->script, "-")) {
-            path = rb_realpath_internal(Qnil, script_name, 1);
+            path = rb_realpath_internal(Qnil, opt->script_name, 1);
 #if UTF8_PATH
             if (uenc != lenc) {
                 path = str_conv_enc(path, uenc, lenc);
@@ -2405,41 +2431,17 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
             }
         }
 
+        bool optimize = !(dump & DUMP_BIT(insns_without_opt));
 
-        if ((*rb_ruby_prism_ptr())) {
-            pm_string_t input;
-            pm_options_t options = { 0 };
-
-            if (strcmp(opt->script, "-") == 0) {
-                int xflag = opt->xflag;
-                VALUE rb_source = open_load_file(opt->script_name, &xflag);
-                opt->xflag = xflag != 0;
-
-                rb_warn("Prism support for streaming code from stdin is not currently supported");
-                pm_string_constant_init(&input, RSTRING_PTR(rb_source), RSTRING_LEN(rb_source));
-                pm_options_filepath_set(&options, RSTRING_PTR(opt->script_name));
-            }
-            else if (opt->e_script) {
-                pm_string_constant_init(&input, RSTRING_PTR(opt->e_script), RSTRING_LEN(opt->e_script));
-                pm_options_filepath_set(&options, "-e");
-            }
-            else {
-                pm_string_mapped_init(&input, RSTRING_PTR(opt->script_name));
-                pm_options_filepath_set(&options, RSTRING_PTR(opt->script_name));
-            }
-
-            iseq = rb_iseq_new_main_prism(&input, &options, path);
-            ruby_opt_init(opt);
-
-            pm_string_free(&input);
-            pm_options_free(&options);
+        if (!ast) {
+            iseq = rb_iseq_new_main_prism(&pm_input, &pm_options, path);
         }
         else {
             rb_binding_t *toplevel_binding;
             GetBindingPtr(rb_const_get(rb_cObject, rb_intern("TOPLEVEL_BINDING")),
                           toplevel_binding);
             const struct rb_block *base_block = toplevel_context(toplevel_binding);
-            iseq = rb_iseq_new_main(&ast->body, opt->script_name, path, vm_block_iseq(base_block), !(dump & DUMP_BIT(insns_without_opt)));
+            iseq = rb_iseq_new_main(&ast->body, opt->script_name, path, vm_block_iseq(base_block), optimize);
             rb_ast_dispose(ast);
         }
     }
diff --git a/test/ruby/test_rubyoptions.rb b/test/ruby/test_rubyoptions.rb
index 38ca119a8f..5fe8877666 100644
--- a/test/ruby/test_rubyoptions.rb
+++ b/test/ruby/test_rubyoptions.rb
@@ -287,17 +287,17 @@ class TestRubyOptions < Test::Unit::TestCase
     end
   end
 
-  def test_parser_flag
-    warning = /compiler based on the Prism parser is currently experimental/
+  PRISM_WARNING = /compiler based on the Prism parser is currently experimental/
 
-    assert_in_out_err(%w(--parser=prism -e) + ["puts :hi"], "", %w(hi), warning)
+  def test_parser_flag
+    assert_in_out_err(%w(--parser=prism -e) + ["puts :hi"], "", %w(hi), PRISM_WARNING)
 
     assert_in_out_err(%w(--parser=parse.y -e) + ["puts :hi"], "", %w(hi), [])
     assert_norun_with_rflag('--parser=parse.y', '--version', "")
 
     assert_in_out_err(%w(--parser=notreal -e) + ["puts :hi"], "", [], /unknown parser notreal/)
 
-    assert_in_out_err(%w(--parser=prism --version), "", /\+PRISM/, warning)
+    assert_in_out_err(%w(--parser=prism --version), "", /\+PRISM/, PRISM_WARNING)
   end
 
   def test_eval
@@ -1140,6 +1140,7 @@ class TestRubyOptions < Test::Unit::TestCase
     assert_norun_with_rflag('--dump=parsetree', '-e', '#frozen-string-literal: true')
     assert_norun_with_rflag('--dump=parsetree+error_tolerant')
     assert_norun_with_rflag('--dump=parse+error_tolerant')
+    assert_in_out_err(%w(--parser=prism --dump=parsetree -e ""), "", /ProgramNode/, PRISM_WARNING, encoding: "UTF-8")
   end
 
   def test_dump_insns_with_rflag
author	Peter Zhu <[email protected]>	2024-06-11 19:43:32 -0400
committer	GitHub <[email protected]>	2024-06-11 16:43:32 -0700
commit	97b1bf9ac11848c2783264d22bf7cdb7f32a21cf (patch)
tree	b8d5259e382badfdf4e857c4e12fb5d4da2f5bee
parent	d1869cfb852cf95b5a51025c016437ab46b12104 (diff)