[ruby/prism] Fix on_words_sep for ripper translator with newlines

Ripper emits a token each per line.

https://github.com/ruby/prism/commit/4b5c9f5437
This commit is contained in:
Earlopain 2026-01-26 12:16:34 +01:00 committed by git
parent e410d938fa
commit 5f25420918
3 changed files with 9 additions and 15 deletions

View File

@ -657,6 +657,15 @@ module Prism
IgnoreStateToken.new([[lineno, column], event, value, lex_state])
when :on_embexpr_end
IgnoreStateToken.new([[lineno, column], event, value, lex_state])
when :on_words_sep
# Ripper emits one token each per line.
lines = value.lines
lines[0...-1].each do |whitespace|
tokens << Token.new([[lineno, column], event, whitespace, lex_state])
lineno += 1
column = 0
end
Token.new([[lineno, column], event, lines.last, lex_state])
when :on_regexp_end
# On regex end, Ripper scans and then sets end state, so the ripper
# lexed output is begin, when it should be end. prism sets lex state

View File

@ -26,13 +26,6 @@ module Prism
results << token
previous = token
end
when :on_words_sep
if previous[1] == :on_words_sep
previous[2] << token[2]
else
results << token
previous = token
end
else
results << token
previous = token

View File

@ -62,18 +62,10 @@ module Prism
]
omitted_lex = [
"comments.txt",
"heredoc_percent_q_newline_delimiter.txt",
"heredoc_with_escaped_newline_at_start.txt",
"heredocs_with_fake_newlines.txt",
"indented_file_end.txt",
"seattlerb/TestRubyParserShared.txt",
"seattlerb/class_comments.txt",
"seattlerb/module_comments.txt",
"seattlerb/parse_line_block_inline_comment_leading_newlines.txt",
"seattlerb/parse_line_block_inline_multiline_comment.txt",
"spanning_heredoc_newlines.txt",
"strings.txt",
"whitequark/dedenting_heredoc.txt",
"whitequark/procarg0.txt",
]