-# frozen_string_literal: true

-

-return if RUBY_ENGINE != "ruby"

-

-require_relative "test_helper"

-

-module Prism

- class EncodingTest < TestCase

- codepoints_1byte = 0...0x100

- encodings = {

- Encoding::ASCII_8BIT => codepoints_1byte,

- Encoding::US_ASCII => codepoints_1byte

- }

-

- if !ENV["PRISM_BUILD_MINIMAL"]

- encodings[Encoding::Windows_1253] = codepoints_1byte

- end

-

- # By default we don't test every codepoint in these encodings because it

- # takes a very long time.

- if ENV["PRISM_TEST_ALL_ENCODINGS"]

- codepoints_2bytes = 0...0x10000

- codepoints_unicode = (0...0x110000)

-

- codepoints_eucjp = [

- *(0...0x10000),

- *(0...0x10000).map { |bytes| bytes | 0x8F0000 }

- ]

-

- codepoints_emacs_mule = [

- *(0...0x80),

- *((0x81...0x90).flat_map { |byte1| (0x90...0x100).map { |byte2| byte1 << 8 | byte2 } }),

- *((0x90...0x9C).flat_map { |byte1| (0xA0...0x100).flat_map { |byte2| (0xA0...0x100).flat_map { |byte3| byte1 << 16 | byte2 << 8 | byte3 } } }),

- *((0xF0...0xF5).flat_map { |byte2| (0xA0...0x100).flat_map { |byte3| (0xA0...0x100).flat_map { |byte4| 0x9C << 24 | byte3 << 16 | byte3 << 8 | byte4 } } }),

- ]

-

- codepoints_gb18030 = [

- *(0...0x80),

- *((0x81..0xFE).flat_map { |byte1| (0x40...0x100).map { |byte2| byte1 << 8 | byte2 } }),

- *((0x81..0xFE).flat_map { |byte1| (0x30...0x40).flat_map { |byte2| (0x81..0xFE).flat_map { |byte3| (0x2F...0x41).map { |byte4| byte1 << 24 | byte2 << 16 | byte3 << 8 | byte4 } } } }),

- ]

-

- codepoints_euc_tw = [

- *(0..0x7F),

- *(0xA1..0xFF).flat_map { |byte1| (0xA1..0xFF).map { |byte2| (byte1 << 8) | byte2 } },

- *(0xA1..0xB0).flat_map { |byte2| (0xA1..0xFF).flat_map { |byte3| (0xA1..0xFF).flat_map { |byte4| 0x8E << 24 | byte2 << 16 | byte3 << 8 | byte4 } } }

- ]

-

- encodings.merge!(

- Encoding::CP850 => codepoints_1byte,

- Encoding::CP852 => codepoints_1byte,

- Encoding::CP855 => codepoints_1byte,

- Encoding::GB1988 => codepoints_1byte,

- Encoding::IBM437 => codepoints_1byte,

- Encoding::IBM720 => codepoints_1byte,

- Encoding::IBM737 => codepoints_1byte,

- Encoding::IBM775 => codepoints_1byte,

- Encoding::IBM852 => codepoints_1byte,

- Encoding::IBM855 => codepoints_1byte,

- Encoding::IBM857 => codepoints_1byte,

- Encoding::IBM860 => codepoints_1byte,

- Encoding::IBM861 => codepoints_1byte,

- Encoding::IBM862 => codepoints_1byte,

- Encoding::IBM863 => codepoints_1byte,

- Encoding::IBM864 => codepoints_1byte,

- Encoding::IBM865 => codepoints_1byte,

- Encoding::IBM866 => codepoints_1byte,

- Encoding::IBM869 => codepoints_1byte,

- Encoding::ISO_8859_1 => codepoints_1byte,

- Encoding::ISO_8859_2 => codepoints_1byte,

- Encoding::ISO_8859_3 => codepoints_1byte,

- Encoding::ISO_8859_4 => codepoints_1byte,

- Encoding::ISO_8859_5 => codepoints_1byte,

- Encoding::ISO_8859_6 => codepoints_1byte,

- Encoding::ISO_8859_7 => codepoints_1byte,

- Encoding::ISO_8859_8 => codepoints_1byte,

- Encoding::ISO_8859_9 => codepoints_1byte,

- Encoding::ISO_8859_10 => codepoints_1byte,

- Encoding::ISO_8859_11 => codepoints_1byte,

- Encoding::ISO_8859_13 => codepoints_1byte,

- Encoding::ISO_8859_14 => codepoints_1byte,

- Encoding::ISO_8859_15 => codepoints_1byte,

- Encoding::ISO_8859_16 => codepoints_1byte,

- Encoding::KOI8_R => codepoints_1byte,

- Encoding::KOI8_U => codepoints_1byte,

- Encoding::MACCENTEURO => codepoints_1byte,

- Encoding::MACCROATIAN => codepoints_1byte,

- Encoding::MACCYRILLIC => codepoints_1byte,

- Encoding::MACGREEK => codepoints_1byte,

- Encoding::MACICELAND => codepoints_1byte,

- Encoding::MACROMAN => codepoints_1byte,

- Encoding::MACROMANIA => codepoints_1byte,

- Encoding::MACTHAI => codepoints_1byte,

- Encoding::MACTURKISH => codepoints_1byte,

- Encoding::MACUKRAINE => codepoints_1byte,

- Encoding::TIS_620 => codepoints_1byte,

- Encoding::Windows_1250 => codepoints_1byte,

- Encoding::Windows_1251 => codepoints_1byte,

- Encoding::Windows_1252 => codepoints_1byte,

- Encoding::Windows_1254 => codepoints_1byte,

- Encoding::Windows_1255 => codepoints_1byte,

- Encoding::Windows_1256 => codepoints_1byte,

- Encoding::Windows_1257 => codepoints_1byte,

- Encoding::Windows_1258 => codepoints_1byte,

- Encoding::Windows_874 => codepoints_1byte,

- Encoding::Big5 => codepoints_2bytes,

- Encoding::Big5_HKSCS => codepoints_2bytes,

- Encoding::Big5_UAO => codepoints_2bytes,

- Encoding::CP949 => codepoints_2bytes,

- Encoding::CP950 => codepoints_2bytes,

- Encoding::CP951 => codepoints_2bytes,

- Encoding::EUC_KR => codepoints_2bytes,

- Encoding::GBK => codepoints_2bytes,

- Encoding::GB12345 => codepoints_2bytes,

- Encoding::GB2312 => codepoints_2bytes,

- Encoding::MACJAPANESE => codepoints_2bytes,

- Encoding::Shift_JIS => codepoints_2bytes,

- Encoding::SJIS_DoCoMo => codepoints_2bytes,

- Encoding::SJIS_KDDI => codepoints_2bytes,

- Encoding::SJIS_SoftBank => codepoints_2bytes,

- Encoding::Windows_31J => codepoints_2bytes,

- Encoding::UTF_8 => codepoints_unicode,

- Encoding::UTF8_MAC => codepoints_unicode,

- Encoding::UTF8_DoCoMo => codepoints_unicode,

- Encoding::UTF8_KDDI => codepoints_unicode,

- Encoding::UTF8_SoftBank => codepoints_unicode,

- Encoding::CESU_8 => codepoints_unicode,

- Encoding::CP51932 => codepoints_eucjp,

- Encoding::EUC_JP => codepoints_eucjp,

- Encoding::EUCJP_MS => codepoints_eucjp,

- Encoding::EUC_JIS_2004 => codepoints_eucjp,

- Encoding::EMACS_MULE => codepoints_emacs_mule,

- Encoding::STATELESS_ISO_2022_JP => codepoints_emacs_mule,

- Encoding::STATELESS_ISO_2022_JP_KDDI => codepoints_emacs_mule,

- Encoding::GB18030 => codepoints_gb18030,

- Encoding::EUC_TW => codepoints_euc_tw

- )

- end

-

- # These test that we're correctly parsing codepoints for each alias of each

- # encoding that prism supports.

- encodings.each do |encoding, range|

- (encoding.names - %w[external internal filesystem locale]).each do |name|

- define_method(:"test_encoding_#{name}") do

- assert_encoding(encoding, name, range)

- end

- end

- end

-

- # These test that we're correctly setting the flags on strings for each

- # encoding that prism supports.

- escapes = ["\\x00", "\\x7F", "\\x80", "\\xFF", "\\u{00}", "\\u{7F}", "\\u{80}", "\\M-\\C-?"]

- escapes = escapes.concat(escapes.product(escapes).map(&:join))

- symbols = [:a, :ą, :+]

- regexps = [/a/, /ą/, //]

-

- encodings.each_key do |encoding|

- define_method(:"test_encoding_flags_#{encoding.name}") do

- assert_encoding_flags(encoding, escapes)

- end

-

- define_method(:"test_symbol_encoding_flags_#{encoding.name}") do

- assert_symbol_encoding_flags(encoding, symbols)

- end

-

- define_method(:"test_symbol_character_escape_encoding_flags_#{encoding.name}") do

- assert_symbol_character_escape_encoding_flags(encoding, escapes)

- end

-

- define_method(:"test_regular_expression_encoding_flags_#{encoding.name}") do

- assert_regular_expression_encoding_flags(encoding, regexps.map(&:inspect))

- end

-

- define_method(:"test_regular_expression_escape_encoding_flags_#{encoding.name}") do

- assert_regular_expression_encoding_flags(encoding, escapes.map { |e| "/#{e}/" })

- end

- end

-

- encoding_modifiers = { ascii_8bit: "n", utf_8: "u", euc_jp: "e", windows_31j: "s" }

- regexp_sources = ["abc", "garçon", "\\x80", "gar\\xC3\\xA7on", "gar\\u{E7}on", "abc\\u{FFFFFF}", "\\x80\\u{80}" ]

-

- encoding_modifiers.each_value do |modifier|

- encodings.each_key do |encoding|

- define_method(:"test_regular_expression_encoding_modifiers_/#{modifier}_#{encoding.name}") do

- assert_regular_expression_encoding_flags(

- encoding,

- regexp_sources.product(encoding_modifiers.values).map { |r, modifier| "/#{r}/#{modifier}" }

- )

- end

- end

- end

-

- def test_coding

- result = Prism.parse("# coding: utf-8\n'string'")

- actual = result.value.statements.body.first.unescaped.encoding

- assert_equal Encoding.find("utf-8"), actual

- end

-

- def test_coding_with_whitespace

- result = Prism.parse("# coding \t \r \v : \t \v \r ascii-8bit \n'string'")

- actual = result.value.statements.body.first.unescaped.encoding

- assert_equal Encoding.find("ascii-8bit"), actual

- end

-

- def test_emacs_style

- result = Prism.parse("# -*- coding: utf-8 -*-\n'string'")

- actual = result.value.statements.body.first.unescaped.encoding

- assert_equal Encoding.find("utf-8"), actual

- end

-

- def test_utf_8_variations

- %w[

- utf-8-unix

- utf-8-dos

- utf-8-mac

- utf-8-*

- ].each do |encoding|

- result = Prism.parse("# coding: #{encoding}\n'string'")

- actual = result.value.statements.body.first.unescaped.encoding

- assert_equal Encoding.find("utf-8"), actual

- end

- end

-

- def test_first_lexed_token

- encoding = Prism.lex("# encoding: ascii-8bit").value[0][0].value.encoding

- assert_equal Encoding.find("ascii-8bit"), encoding

- end

-

- if !ENV["PRISM_BUILD_MINIMAL"]

- # This test may be a little confusing. Basically when we use our strpbrk,

- # it takes into account the encoding of the file.

- def test_strpbrk_multibyte

- result = Prism.parse(<<~RUBY)

- # encoding: Shift_JIS

- %w[\x81\x5c]

- RUBY

-

- assert(result.errors.empty?)

- assert_equal(

- (+"\x81\x5c").force_encoding(Encoding::Shift_JIS),

- result.value.statements.body.first.elements.first.unescaped

- )

- end

-

- def test_slice_encoding

- slice = Prism.parse("# encoding: Shift_JIS\nア").value.slice

- assert_equal (+"ア").force_encoding(Encoding::SHIFT_JIS), slice

- assert_equal Encoding::SHIFT_JIS, slice.encoding

- end

-

- def test_multibyte_escapes

- [

- ["'", "'"],

- ["\"", "\""],

- ["`", "`"],

- ["/", "/"],

- ["<<'HERE'\n", "\nHERE"],

- ["<<-HERE\n", "\nHERE"]

- ].each do |opening, closing|

- assert Prism.parse_success?("# encoding: shift_jis\n'\\\x82\xA0'\n")

- end

- end

- end

-

- private

-

- class ConstantContext < BasicObject

- def self.const_missing(const)

- const

- end

- end

-

- def constant_context

- ConstantContext.new

- end

-

- class IdentifierContext < BasicObject

- def method_missing(name, *)

- name

- end

- end

-

- def identifier_context

- IdentifierContext.new

- end

-

- def assert_encoding_constant(name, character)

- source = "# encoding: #{name}\n#{character}"

- expected = constant_context.instance_eval(source)

-

- result = Prism.parse(source)

- assert result.success?

-

- actual = result.value.statements.body.last

- assert_kind_of ConstantReadNode, actual

- assert_equal expected, actual.name

- end

-

- def assert_encoding_identifier(name, character)

- source = "# encoding: #{name}\n#{character}"

- expected = identifier_context.instance_eval(source)

-

- result = Prism.parse(source)

- assert result.success?

-

- actual = result.value.statements.body.last

- assert_kind_of CallNode, actual

- assert_equal expected, actual.name

- end

-

- # Check that we can properly parse every codepoint in the given encoding.

- def assert_encoding(encoding, name, range)

- # I'm not entirely sure, but I believe these codepoints are incorrect in

- # their parsing in CRuby. They all report as matching `[[:lower:]]` but

- # then they are parsed as constants. This is because CRuby determines if

- # an identifier is a constant or not by case folding it down to lowercase

- # and checking if there is a difference. And even though they report

- # themselves as lowercase, their case fold is different. I have reported

- # this bug upstream.

- case encoding

- when Encoding::UTF_8, Encoding::UTF_8_MAC, Encoding::UTF8_DoCoMo, Encoding::UTF8_KDDI, Encoding::UTF8_SoftBank, Encoding::CESU_8

- range = range.to_a - [

- 0x01c5, 0x01c8, 0x01cb, 0x01f2, 0x1f88, 0x1f89, 0x1f8a, 0x1f8b,

- 0x1f8c, 0x1f8d, 0x1f8e, 0x1f8f, 0x1f98, 0x1f99, 0x1f9a, 0x1f9b,

- 0x1f9c, 0x1f9d, 0x1f9e, 0x1f9f, 0x1fa8, 0x1fa9, 0x1faa, 0x1fab,

- 0x1fac, 0x1fad, 0x1fae, 0x1faf, 0x1fbc, 0x1fcc, 0x1ffc,

- ]

- when Encoding::Windows_1253

- range = range.to_a - [0xb5]

- end

-

- range.each do |codepoint|

- character = codepoint.chr(encoding)

-

- if character.match?(/[[:alpha:]]/)

- if character.match?(/[[:upper:]]/)

- assert_encoding_constant(name, character)

- else

- assert_encoding_identifier(name, character)

- end

- elsif character.match?(/[[:alnum:]]/)

- assert_encoding_identifier(name, "_#{character}")

- else

- next if ["/", "{"].include?(character)

-

- source = "# encoding: #{name}\n/(?##{character})/\n"

- assert Prism.parse(source).success?, "Expected #{source.inspect} to parse successfully."

- end

- rescue RangeError

- source = "# encoding: #{name}\n\\x#{codepoint.to_s(16)}"

- refute Prism.parse(source).success?

- end

- end

-

- def assert_encoding_flags(encoding, escapes)

- escapes.each do |escaped|

- source = "# encoding: #{encoding.name}\n\"#{escaped}\""

-

- expected =

- begin

- eval(source).encoding

- rescue SyntaxError => error

- if error.message.include?("UTF-8 mixed within")

- error.message[/: (.+?)\n/, 1]

- else

- raise

- end

- end

-

- actual =

- Prism.parse(source).then do |result|

- if result.success?

- string = result.value.statements.body.first

-

- if string.forced_utf8_encoding?

- Encoding::UTF_8

- elsif string.forced_binary_encoding?

- Encoding::ASCII_8BIT

- else

- encoding

- end

- else

- error = result.errors.first

-

- if error.message.include?("mixed")

- error.message

- else

- raise error.message

- end

- end

- end

-

- assert_equal expected, actual

- end

- end

-

- # Test Symbol literals without any interpolation or escape sequences.

- def assert_symbol_encoding_flags(encoding, symbols)

- symbols.each do |symbol|

- source = "# encoding: #{encoding.name}\n#{symbol.inspect}"

-

- expected =

- begin

- eval(source).encoding

- rescue SyntaxError => error

- unless error.message.include?("invalid multibyte char")

- raise

- end

- end

-

- actual =

- Prism.parse(source).then do |result|

- if result.success?

- symbol = result.value.statements.body.first

-

- if symbol.forced_utf8_encoding?

- Encoding::UTF_8

- elsif symbol.forced_binary_encoding?

- Encoding::ASCII_8BIT

- elsif symbol.forced_us_ascii_encoding?

- Encoding::US_ASCII

- else

- encoding

- end

- else

- error = result.errors.last

-

- unless error.message.include?("invalid symbol")

- raise error.message

- end

- end

- end

-

- assert_equal expected, actual

- end

- end

-

- def assert_symbol_character_escape_encoding_flags(encoding, escapes)

- escapes.each do |escaped|

- source = "# encoding: #{encoding.name}\n:\"#{escaped}\""

-

- expected =

- begin

- eval(source).encoding

- rescue SyntaxError => error

- if error.message.include?("UTF-8 mixed within")

- error.message[/: (.+?)\n/, 1]

- else

- raise

- end

- end

-

- actual =

- Prism.parse(source).then do |result|

- if result.success?

- symbol = result.value.statements.body.first

-

- if symbol.forced_utf8_encoding?

- Encoding::UTF_8

- elsif symbol.forced_binary_encoding?

- Encoding::ASCII_8BIT

- elsif symbol.forced_us_ascii_encoding?

- Encoding::US_ASCII

- else

- encoding

- end

- else

- error = result.errors.first

-

- if error.message.include?("mixed")

- error.message

- else

- raise error.message

- end

- end

- end

-

- assert_equal expected, actual

- end

- end

-

- def assert_regular_expression_encoding_flags(encoding, regexps)

- regexps.each do |regexp|

- regexp_modifier_used = regexp.end_with?("/u") || regexp.end_with?("/e") || regexp.end_with?("/s") || regexp.end_with?("/n")

- source = "# encoding: #{encoding.name}\n#{regexp}"

-

- encoding_errors = ["invalid multibyte char", "escaped non ASCII character in UTF-8 regexp", "differs from source encoding"]

- skipped_errors = ["invalid multibyte escape", "incompatible character encoding", "UTF-8 character in non UTF-8 regexp", "invalid Unicode range", "invalid Unicode list"]

-

- # TODO (nirvdrum 21-Feb-2024): Prism currently does not handle Regexp validation unless modifiers are used. So, skip processing those errors for now: https://github.com/ruby/prism/issues/2104

- unless regexp_modifier_used

- skipped_errors += encoding_errors

- encoding_errors.clear

- end

-

- expected =

- begin

- eval(source).encoding

- rescue SyntaxError => error

- if encoding_errors.find { |e| error.message.include?(e) }

- error.message.split("\n").map { |m| m[/: (.+?)$/, 1] }

- elsif skipped_errors.find { |e| error.message.include?(e) }

- next

- else

- raise

- end

- end

-

- actual =

- Prism.parse(source).then do |result|

- if result.success?

- regexp = result.value.statements.body.first

-

- actual_encoding = if regexp.forced_utf8_encoding?

- Encoding::UTF_8

- elsif regexp.forced_binary_encoding?

- Encoding::ASCII_8BIT

- elsif regexp.forced_us_ascii_encoding?

- Encoding::US_ASCII

- elsif regexp.ascii_8bit?

- Encoding::ASCII_8BIT

- elsif regexp.utf_8?

- Encoding::UTF_8

- elsif regexp.euc_jp?

- Encoding::EUC_JP

- elsif regexp.windows_31j?

- Encoding::Windows_31J

- else

- encoding

- end

-

- if regexp.utf_8? && actual_encoding != Encoding::UTF_8

- raise "expected regexp encoding to be UTF-8 due to '/u' modifier, but got #{actual_encoding.name}"

- elsif regexp.ascii_8bit? && (actual_encoding != Encoding::ASCII_8BIT && actual_encoding != Encoding::US_ASCII)

- raise "expected regexp encoding to be ASCII-8BIT or US-ASCII due to '/n' modifier, but got #{actual_encoding.name}"

- elsif regexp.euc_jp? && actual_encoding != Encoding::EUC_JP

- raise "expected regexp encoding to be EUC-JP due to '/e' modifier, but got #{actual_encoding.name}"

- elsif regexp.windows_31j? && actual_encoding != Encoding::Windows_31J

- raise "expected regexp encoding to be Windows-31J due to '/s' modifier, but got #{actual_encoding.name}"

- end

-

- if regexp.utf_8? && regexp.forced_utf8_encoding?

- raise "the forced_utf8 flag should not be set when the UTF-8 modifier (/u) is used"

- elsif regexp.ascii_8bit? && regexp.forced_binary_encoding?

- raise "the forced_ascii_8bit flag should not be set when the UTF-8 modifier (/u) is used"

- end

-

- actual_encoding

- else

- errors = result.errors.map(&:message)

-

- if errors.last&.include?("UTF-8 mixed within")

- nil

- else

- errors

- end

- end

- end

-

- # TODO (nirvdrum 22-Feb-2024): Remove this workaround once Prism better maps CRuby's error messages.

- # This class of error message is tricky. The part not being compared is a representation of the regexp.

- # Depending on the source encoding and any encoding modifiers being used, CRuby alters how the regexp is represented.

- # Sometimes it's an MBC string. Other times it uses hexadecimal character escapes. And in other cases it uses

- # the long-form Unicode escape sequences. This short-circuit checks that the error message is mostly correct.

- if expected.is_a?(Array) && actual.is_a?(Array)

- if expected.last.start_with?("/.../n has a non escaped non ASCII character in non ASCII-8BIT script:") &&

- actual.last.start_with?("/.../n has a non escaped non ASCII character in non ASCII-8BIT script:")

- expected.last.clear

- actual.last.clear

- end

- end

-

- assert_equal expected, actual

- end

- end

- end

-end