mirror/ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2026-01-27 04:24:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ruby/prism] Correctly expose ripper state

Browse Source 
It is for example used by `irb`, `rdoc`, `syntax_suggest`

https://github.com/ruby/prism/commit/255aeb2485
This commit is contained in:
Earlopain 2026-01-12 14:21:42 +01:00 committed by git
parent d81a11d4e6
commit ee1aa78bee
5 changed files with 87 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
lib/prism/lex_compat.rb
View File

@ -198,58 +198,6 @@ module Prism
"__END__": :on___end__
}.freeze
# Pretty much a 1:1 copy of Ripper::Lexer::State. We list all the available states
# to reimplement to_s without using Ripper.
class State
# Ripper-internal bitflags.
ALL = %i[
BEG END ENDARG ENDFN ARG CMDARG MID FNAME DOT CLASS LABEL LABELED FITEM
].map.with_index.to_h { |name, i| [2 ** i, name] }
ALL[0] = :NONE
ALL.freeze
ALL.each { |value, name| const_set(name, value) }
# :stopdoc:
attr_reader :to_int, :to_s
def initialize(i)
@to_int = i
@to_s = state_name(i)
freeze
end
def [](index)
case index
when 0, :to_int
@to_int
when 1, :to_s
@to_s
else
nil
end
end
alias to_i to_int
alias inspect to_s
def pretty_print(q) q.text(to_s) end
def ==(i) super or to_int == i end
def &(i) self.class.new(to_int & i) end
def |(i) self.class.new(to_int | i) end
def allbits?(i) to_int.allbits?(i) end
def anybits?(i) to_int.anybits?(i) end
def nobits?(i) to_int.nobits?(i) end
# :startdoc:
private
# Convert the state flags into the format exposed by ripper.
def state_name(bits)
ALL.filter_map { |flag, name| name if bits & flag != 0 }.join("|")
end
end
# When we produce tokens, we produce the same arrays that Ripper does.
# However, we add a couple of convenience methods onto them to make them a
# little easier to work with. We delegate all other methods to the array.
@ -300,8 +248,8 @@ module Prism
class IdentToken < Token
def ==(other) # :nodoc:
(self[0...-1] == other[0...-1]) && (
(other[3] == State::LABEL | State::END) ||
(other[3] & (State::ARG | State::CMDARG) != 0)
(other[3] == Translation::Ripper::EXPR_LABEL | Translation::Ripper::EXPR_END) ||
(other[3] & (Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_CMDARG) != 0)
)
end
end
@ -312,8 +260,8 @@ module Prism
def ==(other) # :nodoc:
return false unless self[0...-1] == other[0...-1]
if self[3] == State::ARG | State::LABELED
other[3] & State::ARG | State::LABELED != 0
if self[3] == Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_LABELED
other[3] & Translation::Ripper::EXPR_ARG | Translation::Ripper::EXPR_LABELED != 0
else
self[3] == other[3]
end
@ -331,8 +279,8 @@ module Prism
class ParamToken < Token
def ==(other) # :nodoc:
(self[0...-1] == other[0...-1]) && (
(other[3] == State::END) ||
(other[3] == State::END | State::LABEL)
(other[3] == Translation::Ripper::EXPR_END) ||
(other[3] == Translation::Ripper::EXPR_END | Translation::Ripper::EXPR_LABEL)
)
end
end
@ -727,7 +675,7 @@ module Prism
event = RIPPER.fetch(token.type)
value = token.value
lex_state = State.new(lex_state)
lex_state = Translation::Ripper::Lexer::State.new(lex_state)
token =
case event
@ -741,7 +689,7 @@ module Prism
last_heredoc_end = token.location.end_offset
IgnoreStateToken.new([[lineno, column], event, value, lex_state])
when :on_ident
if lex_state == State::END
if lex_state == Translation::Ripper::EXPR_END
# If we have an identifier that follows a method name like:
#
# def foo bar
@ -751,7 +699,7 @@ module Prism
# yet. We do this more accurately, so we need to allow comparing
# against both END and END|LABEL.
ParamToken.new([[lineno, column], event, value, lex_state])
elsif lex_state == State::END | State::LABEL
elsif lex_state == Translation::Ripper::EXPR_END | Translation::Ripper::EXPR_LABEL
# In the event that we're comparing identifiers, we're going to
# allow a little divergence. Ripper doesn't account for local
# variables introduced through named captures in regexes, and we
@ -791,7 +739,7 @@ module Prism
counter += { on_embexpr_beg: -1, on_embexpr_end: 1 }[current_event] || 0
end
State.new(result_value[current_index][1])
Translation::Ripper::Lexer::State.new(result_value[current_index][1])
else
previous_state
end

1
lib/prism/prism.gemspec
View File

@ -104,6 +104,7 @@ Gem::Specification.new do |spec|
"lib/prism/translation/parser/compiler.rb",
"lib/prism/translation/parser/lexer.rb",
"lib/prism/translation/ripper.rb",
"lib/prism/translation/ripper/lexer.rb",
"lib/prism/translation/ripper/sexp.rb",
"lib/prism/translation/ripper/shim.rb",
"lib/prism/translation/ruby_parser.rb",

25
lib/prism/translation/ripper.rb
View File

@ -424,9 +424,34 @@ module Prism
end
end
autoload :Lexer, "prism/translation/ripper/lexer"
autoload :SexpBuilder, "prism/translation/ripper/sexp"
autoload :SexpBuilderPP, "prism/translation/ripper/sexp"
# :stopdoc:
# This is not part of the public API but used by some gems.
# Ripper-internal bitflags.
LEX_STATE_NAMES = %i[
BEG END ENDARG ENDFN ARG CMDARG MID FNAME DOT CLASS LABEL LABELED FITEM
].map.with_index.to_h { |name, i| [2 ** i, name] }.freeze
private_constant :LEX_STATE_NAMES
LEX_STATE_NAMES.each do |value, key|
const_set("EXPR_#{key}", value)
end
EXPR_NONE = 0
EXPR_VALUE = EXPR_BEG
EXPR_BEG_ANY = EXPR_BEG | EXPR_MID | EXPR_CLASS
EXPR_ARG_ANY = EXPR_ARG | EXPR_CMDARG
EXPR_END_ANY = EXPR_END | EXPR_ENDARG | EXPR_ENDFN
def self.lex_state_name(state)
LEX_STATE_NAMES.filter_map { |flag, name| name if state & flag != 0 }.join("|")
end
# :startdoc:
# The source that is being parsed.
attr_reader :source

46
lib/prism/translation/ripper/lexer.rb Normal file
View File

@ -0,0 +1,46 @@
# frozen_string_literal: true
# :markup: markdown
require_relative "../ripper"
module Prism
module Translation
class Ripper
class Lexer # :nodoc:
# :stopdoc:
class State
attr_reader :to_int, :to_s
def initialize(i)
@to_int = i
@to_s = Ripper.lex_state_name(i)
freeze
end
def [](index)
case index
when 0, :to_int
@to_int
when 1, :to_s
@to_s
else
nil
end
end
alias to_i to_int
alias inspect to_s
def pretty_print(q) q.text(to_s) end
def ==(i) super or to_int == i end
def &(i) self.class.new(to_int & i) end
def |(i) self.class.new(to_int | i) end
def allbits?(i) to_int.allbits?(i) end
def anybits?(i) to_int.anybits?(i) end
def nobits?(i) to_int.nobits?(i) end
end
# :startdoc:
end
end
end
end

11
test/prism/ruby/ripper_test.rb
View File

@ -65,13 +65,12 @@ module Prism
# Check that the hardcoded values don't change without us noticing.
def test_internals
actual = LexCompat::State::ALL
expected = Ripper.constants.select { |name| name.start_with?("EXPR_") }
expected -= %i[EXPR_VALUE EXPR_BEG_ANY EXPR_ARG_ANY EXPR_END_ANY]
actual = Translation::Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort
expected = Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort
assert_equal(expected.size, actual.size)
expected.each do |const_name|
assert_equal(const_name.to_s.delete_prefix("EXPR_").to_sym, actual[Ripper.const_get(const_name)])
assert_equal(expected, actual)
expected.zip(actual).each do |ripper, prism|
assert_equal(Ripper.const_get(ripper), Translation::Ripper.const_get(prism))
end
end