diff --git a/lib/parser/lexer.rl b/lib/parser/lexer.rl index 2eb3882f4..221c956bf 100644 --- a/lib/parser/lexer.rl +++ b/lib/parser/lexer.rl @@ -545,7 +545,7 @@ class Parser::Lexer @newline_s = p } - c_nl = '\n' $ do_nl; + c_nl = ('\r\n' | '\n') $ do_nl; c_space = [ \t\r\f\v]; c_space_nl = c_space | c_nl; @@ -1995,7 +1995,7 @@ class Parser::Lexer # Here we use '\n' instead of w_newline to not modify @newline_s # and eventually properly emit tNL - (c_space* w_space_comment '\n')+ + (c_space* w_space_comment ('\r\n' | '\n'))+ => { if @version < 27 # Ruby before 2.7 doesn't support comments before leading dot. diff --git a/lib/parser/lexer/literal.rb b/lib/parser/lexer/literal.rb index 537d32ea0..f7e49cc9a 100644 --- a/lib/parser/lexer/literal.rb +++ b/lib/parser/lexer/literal.rb @@ -208,7 +208,7 @@ def flush_string end unless @buffer.empty? - emit(:tSTRING_CONTENT, @buffer, @buffer_s, @buffer_e) + emit(:tSTRING_CONTENT, @buffer.gsub("\r\n", "\n"), @buffer_s, @buffer_e) clear_buffer extend_content diff --git a/lib/parser/source/buffer.rb b/lib/parser/source/buffer.rb index ea9c7cf8a..216296986 100644 --- a/lib/parser/source/buffer.rb +++ b/lib/parser/source/buffer.rb @@ -187,7 +187,7 @@ def raw_source=(input) raise ArgumentError, 'Source::Buffer is immutable' end - @source = input.gsub("\r\n".freeze, "\n".freeze).freeze + @source = input.freeze if !@source.ascii_only? && @source.encoding != Encoding::UTF_32LE && diff --git a/test/test_lexer.rb b/test/test_lexer.rb index 59541d1c2..c3393b1fc 100644 --- a/test/test_lexer.rb +++ b/test/test_lexer.rb @@ -1012,9 +1012,9 @@ def test_heredoc_cr :tIDENTIFIER, "a", [0, 1], :tEQL, "=", [2, 3], :tSTRING_BEG, "<<\"", [4, 7], - :tSTRING_CONTENT, "ABCDEF\r\n", [9, 17], - :tSTRING_END, "E", [17, 20], - :tNL, nil, [8, 9]) + :tSTRING_CONTENT, "ABCDEF\r\n", [10, 19], + :tSTRING_END, "E", [19, 23], + :tNL, nil, [9, 10]) end def test_heredoc_with_identifier_ending_newline__19 @@ -3188,9 +3188,9 @@ def test_bug_heredoc_continuation def test_bug_heredoc_cr_lf assert_scanned("<