5 files changed, 35 insertions, 30 deletions
diff --git a/ext/ripper/ripper_init.c.tmpl b/ext/ripper/ripper_init.c.tmpl
index 08a9c4860b..21f29bd79a 100644
--- a/ext/ripper/ripper_init.c.tmpl
+++ b/ext/ripper/ripper_init.c.tmpl
@@ -78,17 +78,18 @@ static const rb_data_type_t parser_data_type = {
     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
 };
 
-static VALUE
+static rb_parser_string_t *
 ripper_lex_get_generic(struct parser_params *p, rb_parser_input_data input, int line_count)
 {
     VALUE src = (VALUE)input;
     VALUE line = rb_funcallv_public(src, id_gets, 0, 0);
-    if (!NIL_P(line) && !RB_TYPE_P(line, T_STRING)) {
+    if (NIL_P(line)) return 0;
+    if (!RB_TYPE_P(line, T_STRING)) {
         rb_raise(rb_eTypeError,
                  "gets returned %"PRIsVALUE" (expected String or nil)",
                  rb_obj_class(line));
     }
-    return line;
+    return rb_str_to_parser_string(p, line);
 }
 
 void
@@ -104,17 +105,19 @@ ripper_compile_error(struct parser_params *p, const char *fmt, ...)
     ripper_error(p);
 }
 
-static VALUE
+static rb_parser_string_t *
 ripper_lex_io_get(struct parser_params *p, rb_parser_input_data input, int line_count)
 {
     VALUE src = (VALUE)input;
-    return rb_io_gets(src);
+    VALUE line = rb_io_gets(src);
+    if (NIL_P(line)) return 0;
+    return rb_str_to_parser_string(p, line);
 }
 
-static VALUE
+static rb_parser_string_t *
 ripper_lex_get_str(struct parser_params *p, rb_parser_input_data input, int line_count)
 {
-    return rb_parser_lex_get_str((struct lex_pointer_string *)input);
+    return rb_parser_lex_get_str(p, (struct lex_pointer_string *)input);
 }
 
 static VALUE
diff --git a/internal/parse.h b/internal/parse.h
index 5d68993017..a7e8e08912 100644
--- a/internal/parse.h
+++ b/internal/parse.h
@@ -54,7 +54,7 @@ rb_parser_t *rb_ruby_parser_set_context(rb_parser_t *p, const struct rb_iseq_str
 void rb_ruby_parser_set_script_lines(rb_parser_t *p);
 void rb_ruby_parser_error_tolerant(rb_parser_t *p);
 void rb_ruby_parser_keep_tokens(rb_parser_t *p);
-typedef VALUE (rb_parser_lex_gets_func)(struct parser_params*, rb_parser_input_data, int);
+typedef rb_parser_string_t*(rb_parser_lex_gets_func)(struct parser_params*, rb_parser_input_data, int);
 rb_ast_t *rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, const char *fname_ptr, long fname_len, rb_encoding *fname_enc, rb_parser_input_data input, int line);
 
 RUBY_SYMBOL_EXPORT_BEGIN
diff --git a/internal/ruby_parser.h b/internal/ruby_parser.h
index f7f859cb70..8e306d18de 100644
--- a/internal/ruby_parser.h
+++ b/internal/ruby_parser.h
@@ -25,7 +25,7 @@ VALUE rb_parser_new(void);
 VALUE rb_parser_compile_string_path(VALUE vparser, VALUE fname, VALUE src, int line);
 VALUE rb_str_new_parser_string(rb_parser_string_t *str);
 VALUE rb_str_new_mutable_parser_string(rb_parser_string_t *str);
-VALUE rb_parser_lex_get_str(struct lex_pointer_string *ptr_str);
+rb_parser_string_t *rb_parser_lex_get_str(struct parser_params *p, struct lex_pointer_string *ptr_str);
 
 VALUE rb_node_str_string_val(const NODE *);
 VALUE rb_node_sym_string_val(const NODE *);
diff --git a/parse.y b/parse.y
index f2b406bce7..5dec8e01f9 100644
--- a/parse.y
+++ b/parse.y
@@ -486,7 +486,7 @@ struct parser_params {
 
     struct {
         rb_strterm_t *strterm;
-        VALUE (*gets)(struct parser_params*,rb_parser_input_data,int);
+        rb_parser_lex_gets_func *gets;
         rb_parser_input_data input;
         parser_string_buffer_t string_buffer;
         rb_parser_string_t *lastline;
@@ -7783,9 +7783,9 @@ yycompile(struct parser_params *p, const char *fname_ptr, long fname_len, rb_enc
 #endif /* !RIPPER */
 
 static rb_encoding *
-must_be_ascii_compatible(struct parser_params *p, VALUE s)
+must_be_ascii_compatible(struct parser_params *p, rb_parser_string_t *s)
 {
-    rb_encoding *enc = rb_enc_get(s);
+    rb_encoding *enc = rb_parser_str_get_encoding(s);
     if (!rb_enc_asciicompat(enc)) {
         rb_raise(rb_eArgError, "invalid source encoding");
     }
@@ -7795,14 +7795,12 @@ must_be_ascii_compatible(struct parser_params *p, VALUE s)
 static rb_parser_string_t *
 lex_getline(struct parser_params *p)
 {
-    rb_parser_string_t *str;
-    VALUE line = (*p->lex.gets)(p, p->lex.input, p->line_count);
-    if (NIL_P(line)) return 0;
-    must_be_ascii_compatible(p, line);
+    rb_parser_string_t *line = (*p->lex.gets)(p, p->lex.input, p->line_count);
+    if (!line) return 0;
     p->line_count++;
-    str = rb_str_to_parser_string(p, line);
-    string_buffer_append(p, str);
-    return str;
+    string_buffer_append(p, line);
+    must_be_ascii_compatible(p, line);
+    return line;
 }
 
 #ifndef RIPPER
diff --git a/ruby_parser.c b/ruby_parser.c
index 085ba99dfa..f9d4e6d59c 100644
--- a/ruby_parser.c
+++ b/ruby_parser.c
@@ -630,8 +630,8 @@ rb_parser_keep_tokens(VALUE vparser)
     rb_ruby_parser_keep_tokens(parser->parser_params);
 }
 
-VALUE
-rb_parser_lex_get_str(struct lex_pointer_string *ptr_str)
+rb_parser_string_t *
+rb_parser_lex_get_str(struct parser_params *p, struct lex_pointer_string *ptr_str)
 {
     char *beg, *end, *start;
     long len;
@@ -641,20 +641,20 @@ rb_parser_lex_get_str(struct lex_pointer_string *ptr_str)
     len = RSTRING_LEN(s);
     start = beg;
     if (ptr_str->ptr) {
-        if (len == ptr_str->ptr) return Qnil;
+        if (len == ptr_str->ptr) return 0;
         beg += ptr_str->ptr;
         len -= ptr_str->ptr;
     }
     end = memchr(beg, '\n', len);
     if (end) len = ++end - beg;
     ptr_str->ptr += len;
-    return rb_str_subseq(s, beg - start, len);
+    return rb_str_to_parser_string(p, rb_str_subseq(s, beg - start, len));
 }
 
-static VALUE
+static rb_parser_string_t *
 lex_get_str(struct parser_params *p, rb_parser_input_data input, int line_count)
 {
-    return rb_parser_lex_get_str((struct lex_pointer_string *)input);
+    return rb_parser_lex_get_str(p, (struct lex_pointer_string *)input);
 }
 
 static void parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *lines);
@@ -716,15 +716,16 @@ parser_compile_string(struct ruby_parser *parser, const char *f, VALUE s, int li
 
 VALUE rb_io_gets_internal(VALUE io);
 
-static VALUE
+static rb_parser_string_t *
 lex_io_gets(struct parser_params *p, rb_parser_input_data input, int line_count)
 {
     VALUE io = (VALUE)input;
-
-    return rb_io_gets_internal(io);
+    VALUE line = rb_io_gets_internal(io);
+    if (NIL_P(line)) return 0;
+    return rb_str_to_parser_string(p, line);
 }
 
-static VALUE
+static rb_parser_string_t *
 lex_gets_array(struct parser_params *p, rb_parser_input_data data, int index)
 {
     VALUE array = (VALUE)data;
@@ -734,8 +735,11 @@ lex_gets_array(struct parser_params *p, rb_parser_input_data data, int index)
         if (!rb_enc_asciicompat(rb_enc_get(str))) {
             rb_raise(rb_eArgError, "invalid source encoding");
         }
+        return rb_str_to_parser_string(p, str);
+    }
+    else {
+        return 0;
     }
-    return str;
 }
 
 static rb_ast_t*