diff options
| author | Earlopain <[email protected]> | 2024-12-21 22:00:56 +0100 |
|---|---|---|
| committer | Kevin Newton <[email protected]> | 2025-01-11 19:09:05 -0500 |
| commit | 7cbaa3b9298b4ab5027d75a7317ca43a9e745c16 (patch) | |
| tree | 0cf1a458aed88233769a577b76fee441bea35be3 | |
| parent | d597118b3f1f03f56ed723aac2b3ec37b75fdb26 (diff) | |
[ruby/prism] Fix an incompatibility with the parser translator
The offset cache contains an entry for each byte so it can't be accessed via the string length.
Adds tests for all variants except for this:
```
"fo
o" "ba
’"
```
For some reason, this still has the wrong offset.
https://github.com/ruby/prism/commit/a651126458
| -rw-r--r-- | lib/prism/translation/parser/compiler.rb | 8 | ||||
| -rw-r--r-- | lib/prism/translation/parser/lexer.rb | 2 | ||||
| -rw-r--r-- | test/prism/fixtures/dstring.txt | 3 | ||||
| -rw-r--r-- | test/prism/fixtures/dsym_str.txt | 3 | ||||
| -rw-r--r-- | test/prism/fixtures/xstring.txt | 3 | ||||
| -rw-r--r-- | test/prism/ruby/ruby_parser_test.rb | 1 |
6 files changed, 15 insertions, 5 deletions
diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb index 0a95e44f85..3f0294bf62 100644 --- a/lib/prism/translation/parser/compiler.rb +++ b/lib/prism/translation/parser/compiler.rb @@ -1091,7 +1091,7 @@ module Prism start_offset = node.content_loc.start_offset node.unescaped.lines.map do |line| - end_offset = start_offset + line.length + end_offset = start_offset + line.bytesize offsets = srange_offsets(start_offset, end_offset) start_offset = end_offset @@ -1692,7 +1692,7 @@ module Prism start_offset = node.content_loc.start_offset [content_lines, unescaped_lines].transpose.map do |content_line, unescaped_line| - end_offset = start_offset + content_line.length + end_offset = start_offset + content_line.bytesize offsets = srange_offsets(start_offset, end_offset) start_offset = end_offset @@ -1747,7 +1747,7 @@ module Prism start_offset = node.value_loc.start_offset node.value.lines.map do |line| - end_offset = start_offset + line.length + end_offset = start_offset + line.bytesize offsets = srange_offsets(start_offset, end_offset) start_offset = end_offset @@ -1890,7 +1890,7 @@ module Prism start_offset = node.content_loc.start_offset node.unescaped.lines.map do |line| - end_offset = start_offset + line.length + end_offset = start_offset + line.bytesize offsets = srange_offsets(start_offset, end_offset) start_offset = end_offset diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb index db7dbb1c87..61e22159a1 100644 --- a/lib/prism/translation/parser/lexer.rb +++ b/lib/prism/translation/parser/lexer.rb @@ -343,7 +343,7 @@ module Prism adjustment = 0 end - end_offset = start_offset + adjusted_line.length + adjustment + end_offset = start_offset + adjusted_line.bytesize + adjustment tokens << [:tSTRING_CONTENT, [adjusted_line, Range.new(source_buffer, offset_cache[start_offset], offset_cache[end_offset])]] start_offset = end_offset end diff --git a/test/prism/fixtures/dstring.txt b/test/prism/fixtures/dstring.txt index 085e0c6852..69019662c1 100644 --- a/test/prism/fixtures/dstring.txt +++ b/test/prism/fixtures/dstring.txt @@ -27,3 +27,6 @@ foo\\\\ " foo\\\\\ " + +" +’" diff --git a/test/prism/fixtures/dsym_str.txt b/test/prism/fixtures/dsym_str.txt index ee68dde88d..0af0a8ddaf 100644 --- a/test/prism/fixtures/dsym_str.txt +++ b/test/prism/fixtures/dsym_str.txt @@ -1,2 +1,5 @@ :"foo bar" + +:" +’" diff --git a/test/prism/fixtures/xstring.txt b/test/prism/fixtures/xstring.txt index 7ec09468d8..bb3d1c6ee1 100644 --- a/test/prism/fixtures/xstring.txt +++ b/test/prism/fixtures/xstring.txt @@ -11,3 +11,6 @@ `` %x{} + +` +’` diff --git a/test/prism/ruby/ruby_parser_test.rb b/test/prism/ruby/ruby_parser_test.rb index a13daeeb84..fe410c8144 100644 --- a/test/prism/ruby/ruby_parser_test.rb +++ b/test/prism/ruby/ruby_parser_test.rb @@ -46,6 +46,7 @@ module Prism # https://github.com/seattlerb/ruby_parser/issues/344 failures = [ "alias.txt", + "dsym_str.txt", "dos_endings.txt", "heredocs_with_ignored_newlines.txt", "method_calls.txt", |