mirror/ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2026-01-26 12:14:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ruby/prism] Remove unneeded lex_compat token types

Browse Source 
These are either fixed in prism or ruby/ripper itself.

https://github.com/ruby/prism/commit/41c7c126b2
This commit is contained in:
Earlopain 2026-01-24 16:51:30 +01:00 committed by git
parent f7bc28d824
commit 985b58a4ed
1 changed files with 0 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
lib/prism/lex_compat.rb
View File

@ -247,50 +247,6 @@ module Prism
end
end
# Ident tokens for the most part are exactly the same, except sometimes we
# know an ident is a local when ripper doesn't (when they are introduced
# through named captures in regular expressions). In that case we don't
# compare the state.
class IdentToken < Token
def ==(other) # :nodoc:
(self[0...-1] == other[0...-1]) && (
(other[3] == Translation::Ripper::EXPR_LABEL | Translation::Ripper::EXPR_END) ||
(other[3] & (Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_CMDARG) != 0)
)
end
end
# Ignored newlines can occasionally have a LABEL state attached to them, so
# we compare the state differently here.
class IgnoredNewlineToken < Token
def ==(other) # :nodoc:
return false unless self[0...-1] == other[0...-1]
if self[3] == Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_LABELED
other[3] & Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_LABELED != 0
else
self[3] == other[3]
end
end
end
# If we have an identifier that follows a method name like:
#
# def foo bar
#
# then Ripper will mark bar as END|LABEL if there is a local in a parent
# scope named bar because it hasn't pushed the local table yet. We do this
# more accurately, so we need to allow comparing against both END and
# END|LABEL.
class ParamToken < Token
def ==(other) # :nodoc:
(self[0...-1] == other[0...-1]) && (
(other[3] == Translation::Ripper::EXPR_END) ||
(other[3] == Translation::Ripper::EXPR_END | Translation::Ripper::EXPR_LABEL)
)
end
end
# A heredoc in this case is a list of tokens that belong to the body of the
# heredoc that should be appended onto the list of tokens when the heredoc
# closes.
@ -699,33 +655,8 @@ module Prism
# want to bother comparing the state on them.
last_heredoc_end = token.location.end_offset
IgnoreStateToken.new([[lineno, column], event, value, lex_state])
when :on_ident
if lex_state == Translation::Ripper::EXPR_END
# If we have an identifier that follows a method name like:
#
# def foo bar
#
# then Ripper will mark bar as END|LABEL if there is a local in a
# parent scope named bar because it hasn't pushed the local table
# yet. We do this more accurately, so we need to allow comparing
# against both END and END|LABEL.
ParamToken.new([[lineno, column], event, value, lex_state])
elsif lex_state == Translation::Ripper::EXPR_END | Translation::Ripper::EXPR_LABEL
# In the event that we're comparing identifiers, we're going to
# allow a little divergence. Ripper doesn't account for local
# variables introduced through named captures in regexes, and we
# do, which accounts for this difference.
IdentToken.new([[lineno, column], event, value, lex_state])
else
Token.new([[lineno, column], event, value, lex_state])
end
when :on_embexpr_end
IgnoreStateToken.new([[lineno, column], event, value, lex_state])
when :on_ignored_nl
# Ignored newlines can occasionally have a LABEL state attached to
# them which doesn't actually impact anything. We don't mirror that
# state so we ignored it.
IgnoredNewlineToken.new([[lineno, column], event, value, lex_state])
when :on_regexp_end
# On regex end, Ripper scans and then sets end state, so the ripper
# lexed output is begin, when it should be end. prism sets lex state