Merge rdoc-6.1.0.beta2

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@65106 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
author: aycabta <aycabta@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> 2018-10-17 06:28:20 +0000
committer: aycabta <aycabta@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> 2018-10-17 06:28:20 +0000
commit: 1b43644edc85a93bfc9228588c065c87f975cd93 (patch)
tree: ef3a60d0cbe73d15b39f6160853df9a23239a574 /lib/rdoc/parser
parent: 2a59b579fed2fd49973ca73890f182299262909f (diff)
3 files changed, 84 insertions, 80 deletions
diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb
index 773a830450..cc7a87976c 100644
--- a/lib/rdoc/parser/ripper_state_lex.rb
+++ b/lib/rdoc/parser/ripper_state_lex.rb
@@ -50,7 +50,7 @@ class RDoc::Parser::RipperStateLex
         @continue = false
         @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
       end
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_ignored_nl(tok, data)
@@ -61,7 +61,7 @@ class RDoc::Parser::RipperStateLex
         @continue = false
         @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
       end
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_op(tok, data)
@@ -103,7 +103,7 @@ class RDoc::Parser::RipperStateLex
           @lex_state = EXPR_BEG
         end
       end
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_kw(tok, data)
@@ -132,54 +132,54 @@ class RDoc::Parser::RipperStateLex
           @lex_state = EXPR_END
         end
       end
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_tstring_beg(tok, data)
       @lex_state = EXPR_BEG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_tstring_end(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_CHAR(tok, data)
       @lex_state = EXPR_END
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_period(tok, data)
       @lex_state = EXPR_DOT
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_int(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_float(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_rational(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_imaginary(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_symbeg(tok, data)
       @lex_state = EXPR_FNAME
       @continue = true
       @in_fname = true
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     private def on_variables(event, tok, data)
@@ -198,7 +198,7 @@ class RDoc::Parser::RipperStateLex
       else
         @lex_state = EXPR_CMDARG
       end
-      @callback.call(Token.new(lineno, column, event, tok, @lex_state))
+      data << Token.new(lineno, column, event, tok, @lex_state)
     end
 
     def on_ident(tok, data)
@@ -227,32 +227,32 @@ class RDoc::Parser::RipperStateLex
 
     def on_lparen(tok, data)
       @lex_state = EXPR_LABEL | EXPR_BEG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_rparen(tok, data)
       @lex_state = EXPR_ENDFN
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_lbrace(tok, data)
       @lex_state = EXPR_LABEL | EXPR_BEG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_rbrace(tok, data)
       @lex_state = EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_lbracket(tok, data)
       @lex_state = EXPR_LABEL | EXPR_BEG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_rbracket(tok, data)
       @lex_state = EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_const(tok, data)
@@ -264,41 +264,43 @@ class RDoc::Parser::RipperStateLex
       else
         @lex_state = EXPR_CMDARG
       end
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_sp(tok, data)
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_comma(tok, data)
       @lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_comment(tok, data)
       @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_ignored_sp(tok, data)
       @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
+    end
+
+    def on_heredoc_beg(tok, data)
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
+      @lex_state = EXPR_END
+      data
     end
 
     def on_heredoc_end(tok, data)
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
       @lex_state = EXPR_BEG
+      data
     end
 
     def on_default(event, tok, data)
       reset
-      @callback.call(Token.new(lineno, column, event, tok, @lex_state))
-    end
-
-    def each(&block)
-      @callback = block
-      parse
+      data << Token.new(lineno, column, event, tok, @lex_state)
     end
   end unless RIPPER_HAS_LEX_STATE
 
@@ -308,21 +310,17 @@ class RDoc::Parser::RipperStateLex
     end
 
     def on_default(event, tok, data)
-      @callback.call(Token.new(lineno, column, event, tok, state))
-    end
-
-    def each(&block)
-      @callback = block
-      parse
+      data << Token.new(lineno, column, event, tok, state)
     end
   end if RIPPER_HAS_LEX_STATE
 
   def get_squashed_tk
     if @buf.empty?
-      tk = @inner_lex_enumerator.next
+      tk = @tokens.shift
     else
       tk = @buf.shift
     end
+    return nil if tk.nil?
     case tk[:kind]
     when :on_symbeg then
       tk = get_symbol_tk(tk)
@@ -472,7 +470,7 @@ class RDoc::Parser::RipperStateLex
     string = ''
     start_tk = nil
     prev_tk = nil
-    until heredoc_end?(heredoc_name, indent, tk = @inner_lex_enumerator.next) do
+    until heredoc_end?(heredoc_name, indent, tk = @tokens.shift) do
       start_tk = tk unless start_tk
       if (prev_tk.nil? or "\n" == prev_tk[:text][-1]) and 0 != tk[:char_no]
         string = string + (' ' * tk[:char_no])
@@ -555,6 +553,10 @@ class RDoc::Parser::RipperStateLex
         tk[:text] += tk_ahead[:text]
         tk[:kind] = tk_ahead[:kind]
         tk[:state] = tk_ahead[:state]
+      when :on_heredoc_beg, :on_tstring, :on_dstring # frozen/non-frozen string literal
+        tk[:text] += tk_ahead[:text]
+        tk[:kind] = tk_ahead[:kind]
+        tk[:state] = tk_ahead[:state]
       else
         @buf.unshift tk_ahead
       end
@@ -566,11 +568,7 @@ class RDoc::Parser::RipperStateLex
     @buf = []
     @heredoc_queue = []
     @inner_lex = InnerStateLex.new(code)
-    @inner_lex_enumerator = Enumerator.new do |y|
-      @inner_lex.each do |tk|
-        y << tk
-      end
-    end
+    @tokens = @inner_lex.parse([])
   end
 
   def self.parse(code)
diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb
index 8b9ecc1141..97399f87ff 100644
--- a/lib/rdoc/parser/ruby.rb
+++ b/lib/rdoc/parser/ruby.rb
@@ -8,8 +8,6 @@
 #       by Keiju ISHITSUKA (Nippon Rational Inc.)
 #
 
-$TOKEN_DEBUG ||= nil
-
 ##
 # Extracts code elements from a source file returning a TopLevel object
 # containing the constituent file elements.
@@ -267,7 +265,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
         tk = get_tk
 
         if :on_nl === tk then
-          skip_tkspace false
+          skip_tkspace_without_nl
           tk = get_tk
         end
       end
@@ -282,7 +280,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Consumes trailing whitespace from the token stream
 
   def consume_trailing_spaces # :nodoc:
-    skip_tkspace false
+    skip_tkspace_without_nl
   end
 
   ##
@@ -354,7 +352,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       given_name << '::'
     end
 
-    skip_tkspace false
+    skip_tkspace_without_nl
     given_name << name_t[:text]
 
     is_self = name_t[:kind] == :on_op && name_t[:text] == '<<'
@@ -378,7 +376,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       record_location container
 
       get_tk
-      skip_tkspace false
+      skip_tkspace_without_nl
       name_t = get_tk
       unless :on_const == name_t[:kind] || :on_ident == name_t[:kind]
         raise RDoc::Error, "Invalid class or module definition: #{given_name}"
@@ -390,7 +388,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       end
     end
 
-    skip_tkspace false
+    skip_tkspace_without_nl
 
     return [container, name_t, given_name, new_modules]
   end
@@ -410,7 +408,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     res = get_constant
 
-    skip_tkspace false
+    skip_tkspace_without_nl
 
     get_tkread # empty out read buffer
 
@@ -433,7 +431,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def get_constant
     res = ""
-    skip_tkspace false
+    skip_tkspace_without_nl
     tk = get_tk
 
     while tk && ((:on_op == tk[:kind] && '::' == tk[:text]) || :on_const == tk[:kind]) do
@@ -449,7 +447,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Get an included module that may be surrounded by parens
 
   def get_included_module_with_optional_parens
-    skip_tkspace false
+    skip_tkspace_without_nl
     get_tkread
     tk = get_tk
     end_token = get_end_token tk
@@ -685,7 +683,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     if args.size > 0 then
       name = args[0]
       rw = "R"
-      skip_tkspace false
+      skip_tkspace_without_nl
       tk = get_tk
 
       if :on_comma == tk[:kind] then
@@ -935,7 +933,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     line_no = tk[:line_no]
 
     name = tk[:text]
-    skip_tkspace false
+    skip_tkspace_without_nl
 
     return unless name =~ /^\w+$/
 
@@ -961,7 +959,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
           break if nest == 0
         end
       end
-      skip_tkspace false
+      skip_tkspace_without_nl
       is_array_or_hash = true
     end
 
@@ -1298,7 +1296,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     add_token tk
     add_token_listener self
 
-    skip_tkspace false
+    skip_tkspace_without_nl
 
     comment.text = comment.text.sub(/(^# +:?)(singleton-)(method:)/, '\1\3')
     singleton = !!$~
@@ -1487,7 +1485,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def parse_method_name container # :nodoc:
     skip_tkspace
     name_t = get_tk
-    back_tk = skip_tkspace(false)
+    back_tk = skip_tkspace_without_nl
     singleton = false
 
     dot = get_tk
@@ -1575,7 +1573,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_method_or_yield_parameters(method = nil,
                                        modifiers = RDoc::METHOD_MODIFIERS)
-    skip_tkspace false
+    skip_tkspace_without_nl
     tk = get_tk
     end_token = get_end_token tk
     return '' unless end_token
@@ -1648,7 +1646,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     return if  method.block_params
 
-    skip_tkspace false
+    skip_tkspace_without_nl
     read_documentation_modifiers method, RDoc::METHOD_MODIFIERS
   end
 
@@ -1699,19 +1697,19 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses a rescue
 
   def parse_rescue
-    skip_tkspace false
+    skip_tkspace_without_nl
 
     while tk = get_tk
       case tk[:kind]
       when :on_nl, :on_semicolon, :on_comment then
         break
       when :on_comma then
-        skip_tkspace false
+        skip_tkspace_without_nl
 
         get_tk if :on_nl == peek_tk[:kind]
       end
 
-      skip_tkspace false
+      skip_tkspace_without_nl
     end
   end
 
@@ -1784,7 +1782,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
             comment += "\n" unless "\n" == comment_body.chars.to_a.last
 
             if comment_body.size > 1 && "\n" == comment_body.chars.to_a.last then
-              skip_tkspace false # leading spaces
+              skip_tkspace_without_nl # leading spaces
             end
             tk = get_tk
           end
@@ -1968,7 +1966,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     end
 
     loop do
-      skip_tkspace false
+      skip_tkspace_without_nl
 
       tk1 = get_tk
       if tk1.nil? || :on_comma != tk1[:kind] then
@@ -2117,7 +2115,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # See also RDoc::Markup::PreProcess#handle_directive
 
   def read_documentation_modifiers context, allowed
-    skip_tkspace(false)
+    skip_tkspace_without_nl
     directive, value = read_directive allowed
 
     return unless directive
@@ -2195,7 +2193,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # while, until, and for have an optional do
 
   def skip_optional_do_after_expression
-    skip_tkspace false
+    skip_tkspace_without_nl
     tk = get_tk
 
     b_nest = 0
@@ -2227,7 +2225,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       tk = get_tk
     end
 
-    skip_tkspace false
+    skip_tkspace_without_nl
 
     get_tk if peek_tk && :on_kw == peek_tk[:kind] && 'do' == peek_tk[:text]
   end
@@ -2236,9 +2234,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # skip the var [in] part of a 'for' statement
 
   def skip_for_variable
-    skip_tkspace false
+    skip_tkspace_without_nl
     get_tk
-    skip_tkspace false
+    skip_tkspace_without_nl
     tk = get_tk
     unget_tk(tk) unless :on_kw == tk[:kind] and 'in' == tk[:text]
   end
@@ -2257,7 +2255,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def skip_tkspace_comment(skip_nl = true)
     loop do
-      skip_tkspace skip_nl
+      skip_nl ? skip_tkspace : skip_tkspace_without_nl
       next_tk = peek_tk
       return if next_tk.nil? || (:on_comment != next_tk[:kind] and :on_embdoc != next_tk[:kind])
       get_tk
diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb
index 1f621cd32e..681d7166ce 100644
--- a/lib/rdoc/parser/ruby_tools.rb
+++ b/lib/rdoc/parser/ruby_tools.rb
@@ -25,12 +25,10 @@ module RDoc::Parser::RubyTools
         tk = @scanner[@scanner_point]
         @scanner_point += 1
         @read.push tk[:text]
-        puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
       end
     else
       @read.push @unget_read.shift
       tk = @tokens.shift
-      puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
     end
 
     if tk == nil || :on___end__ == tk[:kind]
@@ -111,17 +109,27 @@ module RDoc::Parser::RubyTools
     @scanner_point = 0
   end
 
-  def tk_nl?(tk)
-    :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
+  ##
+  # Skips whitespace tokens including newlines
+
+  def skip_tkspace
+    tokens = []
+
+    while (tk = get_tk) and (:on_sp == tk[:kind] or :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]) do
+      tokens.push(tk)
+    end
+
+    unget_tk(tk)
+    tokens
   end
 
   ##
-  # Skips whitespace tokens including newlines if +skip_nl+ is true
+  # Skips whitespace tokens excluding newlines
 
-  def skip_tkspace(skip_nl = true)
+  def skip_tkspace_without_nl
     tokens = []
 
-    while (tk = get_tk) and (:on_sp == tk[:kind] or (skip_nl and tk_nl?(tk))) do
+    while (tk = get_tk) and :on_sp == tk[:kind] do
       tokens.push(tk)
     end
author	aycabta <aycabta@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>	2018-10-17 06:28:20 +0000
committer	aycabta <aycabta@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>	2018-10-17 06:28:20 +0000
commit	1b43644edc85a93bfc9228588c065c87f975cd93 (patch)
tree	ef3a60d0cbe73d15b39f6160853df9a23239a574 /lib/rdoc/parser
parent	2a59b579fed2fd49973ca73890f182299262909f (diff)