diff options
-rw-r--r-- | lib/prism/translation/parser/compiler.rb | 61 | ||||
-rw-r--r-- | lib/prism/translation/parser/lexer.rb | 6 | ||||
-rw-r--r-- | test/prism/fixtures/dstring.txt | 3 | ||||
-rw-r--r-- | test/prism/fixtures/xstring.txt | 3 | ||||
-rw-r--r-- | test/prism/snapshots/dstring.txt | 95 | ||||
-rw-r--r-- | test/prism/snapshots/dsym_str.txt | 19 | ||||
-rw-r--r-- | test/prism/snapshots/xstring.txt | 78 |
7 files changed, 265 insertions, 0 deletions
diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb index 05c3a64c5b..9fd39ca4a2 100644 --- a/lib/prism/translation/parser/compiler.rb +++ b/lib/prism/translation/parser/compiler.rb @@ -1117,6 +1117,29 @@ module Prism return visit_heredoc(node) { |children, closing| builder.string_compose(token(node.opening_loc), children, closing) } end +<<<<<<< HEAD +======= + parts = if node.parts.one? { |part| part.type == :string_node } + node.parts.flat_map do |node| + if node.type == :string_node && node.unescaped.lines.count >= 2 + start_offset = node.content_loc.start_offset + + node.unescaped.lines.map do |line| + end_offset = start_offset + line.bytesize + offsets = srange_offsets(start_offset, end_offset) + start_offset = end_offset + + builder.string_internal([line, offsets]) + end + else + visit(node) + end + end + else + visit_all(node.parts) + end + +>>>>>>> a651126458 (Fix an incompatibility with the parser translator) builder.string_compose( token(node.opening_loc), string_nodes_from_interpolation(node, node.opening), @@ -1700,7 +1723,19 @@ module Prism if node.content.include?("\n") string_nodes_from_line_continuations(node.unescaped, node.content, node.content_loc.start_offset, node.opening) else +<<<<<<< HEAD [builder.string_internal([node.unescaped, srange(node.content_loc)])] +======= + start_offset = node.content_loc.start_offset + + [content_lines, unescaped_lines].transpose.map do |content_line, unescaped_line| + end_offset = start_offset + content_line.bytesize + offsets = srange_offsets(start_offset, end_offset) + start_offset = end_offset + + builder.string_internal([unescaped_line, offsets]) + end +>>>>>>> a651126458 (Fix an incompatibility with the parser translator) end builder.string_compose( @@ -1744,6 +1779,7 @@ module Prism builder.symbol([node.unescaped, srange(node.location)]) end else +<<<<<<< HEAD parts = if node.value == "" [] @@ -1751,6 +1787,19 @@ module Prism string_nodes_from_line_continuations(node.unescaped, node.value, node.value_loc.start_offset, node.opening) else [builder.string_internal([node.unescaped, srange(node.value_loc)])] +======= + parts = if node.value.lines.one? + [builder.string_internal([node.unescaped, srange(node.value_loc)])] + else + start_offset = node.value_loc.start_offset + + node.value.lines.map do |line| + end_offset = start_offset + line.bytesize + offsets = srange_offsets(start_offset, end_offset) + start_offset = end_offset + + builder.string_internal([line, offsets]) +>>>>>>> a651126458 (Fix an incompatibility with the parser translator) end builder.symbol_compose( @@ -1889,7 +1938,19 @@ module Prism elsif node.content.include?("\n") string_nodes_from_line_continuations(node.unescaped, node.content, node.content_loc.start_offset, node.opening) else +<<<<<<< HEAD [builder.string_internal([node.unescaped, srange(node.content_loc)])] +======= + start_offset = node.content_loc.start_offset + + node.unescaped.lines.map do |line| + end_offset = start_offset + line.bytesize + offsets = srange_offsets(start_offset, end_offset) + start_offset = end_offset + + builder.string_internal([line, offsets]) + end +>>>>>>> a651126458 (Fix an incompatibility with the parser translator) end builder.xstring_compose( diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb index 7db519499f..4f2ea1855a 100644 --- a/lib/prism/translation/parser/lexer.rb +++ b/lib/prism/translation/parser/lexer.rb @@ -422,6 +422,7 @@ module Prism value = trim_heredoc_whitespace(token.value, current_heredoc) end +<<<<<<< HEAD current_string << unescape_string(value, quote_stack.last) if (backslash_count = token.value[/(\\{1,})\n/, 1]&.length).nil? || backslash_count.even? || !interpolation?(quote_stack.last) tokens << [:tSTRING_CONTENT, [current_string, range(start_offset, start_offset + current_length)]] @@ -466,6 +467,11 @@ module Prism current_line = +"" adjustment = 0 end +======= + end_offset = start_offset + adjusted_line.bytesize + adjustment + tokens << [:tSTRING_CONTENT, [adjusted_line, Range.new(source_buffer, offset_cache[start_offset], offset_cache[end_offset])]] + start_offset = end_offset +>>>>>>> a651126458 (Fix an incompatibility with the parser translator) end end next diff --git a/test/prism/fixtures/dstring.txt b/test/prism/fixtures/dstring.txt index 99f6c0dfac..d972b3f71d 100644 --- a/test/prism/fixtures/dstring.txt +++ b/test/prism/fixtures/dstring.txt @@ -29,10 +29,13 @@ foo\\\\\ " " +<<<<<<< HEAD foo\ b\nar #{} " " +======= +>>>>>>> a651126458 (Fix an incompatibility with the parser translator) ’" diff --git a/test/prism/fixtures/xstring.txt b/test/prism/fixtures/xstring.txt index 465a14e84b..1d1bba8e3a 100644 --- a/test/prism/fixtures/xstring.txt +++ b/test/prism/fixtures/xstring.txt @@ -13,9 +13,12 @@ %x{} ` +<<<<<<< HEAD foo\ b\nar ` ` +======= +>>>>>>> a651126458 (Fix an incompatibility with the parser translator) ’` diff --git a/test/prism/snapshots/dstring.txt b/test/prism/snapshots/dstring.txt new file mode 100644 index 0000000000..7c9692fc18 --- /dev/null +++ b/test/prism/snapshots/dstring.txt @@ -0,0 +1,95 @@ +@ ProgramNode (location: (1,0)-(32,4)) +├── flags: ∅ +├── locals: [] +└── statements: + @ StatementsNode (location: (1,0)-(32,4)) + ├── flags: ∅ + └── body: (length: 9) + ├── @ StringNode (location: (1,0)-(2,6)) + │ ├── flags: newline + │ ├── opening_loc: (1,0)-(1,1) = "\"" + │ ├── content_loc: (1,1)-(2,5) = "foo\n bar" + │ ├── closing_loc: (2,5)-(2,6) = "\"" + │ └── unescaped: "foo\n bar" + ├── @ InterpolatedStringNode (location: (4,0)-(5,9)) + │ ├── flags: newline + │ ├── opening_loc: (4,0)-(4,1) = "\"" + │ ├── parts: (length: 2) + │ │ ├── @ StringNode (location: (4,1)-(5,2)) + │ │ │ ├── flags: static_literal, frozen + │ │ │ ├── opening_loc: ∅ + │ │ │ ├── content_loc: (4,1)-(5,2) = "foo\n " + │ │ │ ├── closing_loc: ∅ + │ │ │ └── unescaped: "foo\n " + │ │ └── @ EmbeddedStatementsNode (location: (5,2)-(5,8)) + │ │ ├── flags: ∅ + │ │ ├── opening_loc: (5,2)-(5,4) = "\#{" + │ │ ├── statements: + │ │ │ @ StatementsNode (location: (5,4)-(5,7)) + │ │ │ ├── flags: ∅ + │ │ │ └── body: (length: 1) + │ │ │ └── @ CallNode (location: (5,4)-(5,7)) + │ │ │ ├── flags: variable_call, ignore_visibility + │ │ │ ├── receiver: ∅ + │ │ │ ├── call_operator_loc: ∅ + │ │ │ ├── name: :bar + │ │ │ ├── message_loc: (5,4)-(5,7) = "bar" + │ │ │ ├── opening_loc: ∅ + │ │ │ ├── arguments: ∅ + │ │ │ ├── closing_loc: ∅ + │ │ │ └── block: ∅ + │ │ └── closing_loc: (5,7)-(5,8) = "}" + │ └── closing_loc: (5,8)-(5,9) = "\"" + ├── @ InterpolatedStringNode (location: (7,0)-(9,2)) + │ ├── flags: newline, static_literal + │ ├── opening_loc: ∅ + │ ├── parts: (length: 2) + │ │ ├── @ StringNode (location: (7,0)-(8,2)) + │ │ │ ├── flags: static_literal, frozen + │ │ │ ├── opening_loc: (7,0)-(7,1) = "\"" + │ │ │ ├── content_loc: (7,1)-(8,1) = "fo\no" + │ │ │ ├── closing_loc: (8,1)-(8,2) = "\"" + │ │ │ └── unescaped: "fo\no" + │ │ └── @ StringNode (location: (8,3)-(9,2)) + │ │ ├── flags: static_literal, frozen + │ │ ├── opening_loc: (8,3)-(8,4) = "\"" + │ │ ├── content_loc: (8,4)-(9,1) = "ba\nr" + │ │ ├── closing_loc: (9,1)-(9,2) = "\"" + │ │ └── unescaped: "ba\nr" + │ └── closing_loc: ∅ + ├── @ StringNode (location: (11,0)-(13,1)) + │ ├── flags: newline + │ ├── opening_loc: (11,0)-(11,1) = "\"" + │ ├── content_loc: (11,1)-(13,0) = "\nfoo\\\n" + │ ├── closing_loc: (13,0)-(13,1) = "\"" + │ └── unescaped: "\nfoo" + ├── @ StringNode (location: (15,0)-(17,1)) + │ ├── flags: newline + │ ├── opening_loc: (15,0)-(15,1) = "\"" + │ ├── content_loc: (15,1)-(17,0) = "\nfoo\\\\\n" + │ ├── closing_loc: (17,0)-(17,1) = "\"" + │ └── unescaped: "\nfoo\\\n" + ├── @ StringNode (location: (19,0)-(21,1)) + │ ├── flags: newline + │ ├── opening_loc: (19,0)-(19,1) = "\"" + │ ├── content_loc: (19,1)-(21,0) = "\nfoo\\\\\\\n" + │ ├── closing_loc: (21,0)-(21,1) = "\"" + │ └── unescaped: "\nfoo\\" + ├── @ StringNode (location: (23,0)-(25,1)) + │ ├── flags: newline + │ ├── opening_loc: (23,0)-(23,1) = "\"" + │ ├── content_loc: (23,1)-(25,0) = "\nfoo\\\\\\\\\n" + │ ├── closing_loc: (25,0)-(25,1) = "\"" + │ └── unescaped: "\nfoo\\\\\n" + ├── @ StringNode (location: (27,0)-(29,1)) + │ ├── flags: newline + │ ├── opening_loc: (27,0)-(27,1) = "\"" + │ ├── content_loc: (27,1)-(29,0) = "\nfoo\\\\\\\\\\\n" + │ ├── closing_loc: (29,0)-(29,1) = "\"" + │ └── unescaped: "\nfoo\\\\" + └── @ StringNode (location: (31,0)-(32,4)) + ├── flags: newline + ├── opening_loc: (31,0)-(31,1) = "\"" + ├── content_loc: (31,1)-(32,3) = "\n’" + ├── closing_loc: (32,3)-(32,4) = "\"" + └── unescaped: "\n’" diff --git a/test/prism/snapshots/dsym_str.txt b/test/prism/snapshots/dsym_str.txt new file mode 100644 index 0000000000..629fa9e969 --- /dev/null +++ b/test/prism/snapshots/dsym_str.txt @@ -0,0 +1,19 @@ +@ ProgramNode (location: (1,0)-(5,4)) +├── flags: ∅ +├── locals: [] +└── statements: + @ StatementsNode (location: (1,0)-(5,4)) + ├── flags: ∅ + └── body: (length: 2) + ├── @ SymbolNode (location: (1,0)-(2,6)) + │ ├── flags: newline, static_literal, forced_us_ascii_encoding + │ ├── opening_loc: (1,0)-(1,2) = ":\"" + │ ├── value_loc: (1,2)-(2,5) = "foo\n bar" + │ ├── closing_loc: (2,5)-(2,6) = "\"" + │ └── unescaped: "foo\n bar" + └── @ SymbolNode (location: (4,0)-(5,4)) + ├── flags: newline, static_literal + ├── opening_loc: (4,0)-(4,2) = ":\"" + ├── value_loc: (4,2)-(5,3) = "\n’" + ├── closing_loc: (5,3)-(5,4) = "\"" + └── unescaped: "\n’" diff --git a/test/prism/snapshots/xstring.txt b/test/prism/snapshots/xstring.txt new file mode 100644 index 0000000000..b3a3cb801b --- /dev/null +++ b/test/prism/snapshots/xstring.txt @@ -0,0 +1,78 @@ +@ ProgramNode (location: (1,0)-(16,4)) +├── flags: ∅ +├── locals: [] +└── statements: + @ StatementsNode (location: (1,0)-(16,4)) + ├── flags: ∅ + └── body: (length: 7) + ├── @ XStringNode (location: (1,0)-(1,7)) + │ ├── flags: newline + │ ├── opening_loc: (1,0)-(1,3) = "%x[" + │ ├── content_loc: (1,3)-(1,6) = "foo" + │ ├── closing_loc: (1,6)-(1,7) = "]" + │ └── unescaped: "foo" + ├── @ InterpolatedXStringNode (location: (3,0)-(3,16)) + │ ├── flags: newline + │ ├── opening_loc: (3,0)-(3,1) = "`" + │ ├── parts: (length: 3) + │ │ ├── @ StringNode (location: (3,1)-(3,5)) + │ │ │ ├── flags: static_literal, frozen + │ │ │ ├── opening_loc: ∅ + │ │ │ ├── content_loc: (3,1)-(3,5) = "foo " + │ │ │ ├── closing_loc: ∅ + │ │ │ └── unescaped: "foo " + │ │ ├── @ EmbeddedStatementsNode (location: (3,5)-(3,11)) + │ │ │ ├── flags: ∅ + │ │ │ ├── opening_loc: (3,5)-(3,7) = "\#{" + │ │ │ ├── statements: + │ │ │ │ @ StatementsNode (location: (3,7)-(3,10)) + │ │ │ │ ├── flags: ∅ + │ │ │ │ └── body: (length: 1) + │ │ │ │ └── @ CallNode (location: (3,7)-(3,10)) + │ │ │ │ ├── flags: variable_call, ignore_visibility + │ │ │ │ ├── receiver: ∅ + │ │ │ │ ├── call_operator_loc: ∅ + │ │ │ │ ├── name: :bar + │ │ │ │ ├── message_loc: (3,7)-(3,10) = "bar" + │ │ │ │ ├── opening_loc: ∅ + │ │ │ │ ├── arguments: ∅ + │ │ │ │ ├── closing_loc: ∅ + │ │ │ │ └── block: ∅ + │ │ │ └── closing_loc: (3,10)-(3,11) = "}" + │ │ └── @ StringNode (location: (3,11)-(3,15)) + │ │ ├── flags: static_literal, frozen + │ │ ├── opening_loc: ∅ + │ │ ├── content_loc: (3,11)-(3,15) = " baz" + │ │ ├── closing_loc: ∅ + │ │ └── unescaped: " baz" + │ └── closing_loc: (3,15)-(3,16) = "`" + ├── @ XStringNode (location: (5,0)-(5,5)) + │ ├── flags: newline + │ ├── opening_loc: (5,0)-(5,1) = "`" + │ ├── content_loc: (5,1)-(5,4) = "foo" + │ ├── closing_loc: (5,4)-(5,5) = "`" + │ └── unescaped: "foo" + ├── @ XStringNode (location: (7,0)-(9,1)) + │ ├── flags: newline + │ ├── opening_loc: (7,0)-(7,3) = "%x{" + │ ├── content_loc: (7,3)-(9,0) = "\n foo\n" + │ ├── closing_loc: (9,0)-(9,1) = "}" + │ └── unescaped: "\n foo\n" + ├── @ XStringNode (location: (11,0)-(11,2)) + │ ├── flags: newline + │ ├── opening_loc: (11,0)-(11,1) = "`" + │ ├── content_loc: (11,1)-(11,1) = "" + │ ├── closing_loc: (11,1)-(11,2) = "`" + │ └── unescaped: "" + ├── @ XStringNode (location: (13,0)-(13,4)) + │ ├── flags: newline + │ ├── opening_loc: (13,0)-(13,3) = "%x{" + │ ├── content_loc: (13,3)-(13,3) = "" + │ ├── closing_loc: (13,3)-(13,4) = "}" + │ └── unescaped: "" + └── @ XStringNode (location: (15,0)-(16,4)) + ├── flags: newline + ├── opening_loc: (15,0)-(15,1) = "`" + ├── content_loc: (15,1)-(16,3) = "\n’" + ├── closing_loc: (16,3)-(16,4) = "`" + └── unescaped: "\n’" |