class LL::Lexer
Ragel lexer for LL
grammar files.
Attributes
Public Class Methods
@param [String] data The data to lex. @param [String] file The name of the input file.
# File lib/ll/lexer.rb, line 202 def initialize(data, file = SourceLine::DEFAULT_FILE) @data = data @file = file reset end
Public Instance Methods
Advances through the input and generates the corresponding tokens. Each token is yielded to the supplied block.
@see [#add_token]
# File lib/ll/lexer.rb, line 240 def advance(&block) @block = block data = @data # saves ivar lookups while lexing. ts = nil te = nil cs = self.class.ll_lexer_start act = 0 eof = @data.bytesize p = 0 pe = eof mark = 0 brace_count = 0 start_line = 0 _ll_lexer_eof_trans = self.class.send(:_ll_lexer_eof_trans) _ll_lexer_from_state_actions = self.class.send(:_ll_lexer_from_state_actions) _ll_lexer_index_offsets = self.class.send(:_ll_lexer_index_offsets) _ll_lexer_indicies = self.class.send(:_ll_lexer_indicies) _ll_lexer_key_spans = self.class.send(:_ll_lexer_key_spans) _ll_lexer_to_state_actions = self.class.send(:_ll_lexer_to_state_actions) _ll_lexer_trans_actions = self.class.send(:_ll_lexer_trans_actions) _ll_lexer_trans_keys = self.class.send(:_ll_lexer_trans_keys) _ll_lexer_trans_targs = self.class.send(:_ll_lexer_trans_targs) # line 268 "lib/ll/lexer.rb" begin testEof = false _slen, _trans, _keys, _inds, _acts, _nacts = nil _goto_level = 0 _resume = 10 _eof_trans = 15 _again = 20 _test_eof = 30 _out = 40 while true if _goto_level <= 0 if p == pe _goto_level = _test_eof next end if cs == 0 _goto_level = _out next end end if _goto_level <= _resume case _ll_lexer_from_state_actions[cs] when 6 then # line 1 "NONE" begin ts = p end # line 296 "lib/ll/lexer.rb" end _keys = cs << 1 _inds = _ll_lexer_index_offsets[cs] _slen = _ll_lexer_key_spans[cs] _wide = ( (data.getbyte(p) || 0)) _trans = if ( _slen > 0 && _ll_lexer_trans_keys[_keys] <= _wide && _wide <= _ll_lexer_trans_keys[_keys + 1] ) then _ll_lexer_indicies[ _inds + _wide - _ll_lexer_trans_keys[_keys] ] else _ll_lexer_indicies[ _inds + _slen ] end end if _goto_level <= _eof_trans cs = _ll_lexer_trans_targs[_trans] if _ll_lexer_trans_actions[_trans] != 0 case _ll_lexer_trans_actions[_trans] when 25 then # line 188 "lib/ll/lexer.rl" begin te = p+1 begin brace_count += 1 end end when 26 then # line 190 "lib/ll/lexer.rl" begin te = p+1 begin if brace_count == 1 emit(:T_RUBY, mark, ts, start_line) mark = 0 brace_count = 0 start_line = 0 advance_column cs = 22; else brace_count -= 1 end end end when 23 then # line 206 "lib/ll/lexer.rl" begin te = p+1 end when 3 then # line 215 "lib/ll/lexer.rl" begin te = p+1 begin emit(:T_NAME, ts, te) end end when 4 then # line 216 "lib/ll/lexer.rl" begin te = p+1 begin emit(:T_TERMINALS, ts, te) end end when 2 then # line 217 "lib/ll/lexer.rl" begin te = p+1 begin emit(:T_INNER, ts, te) end end when 1 then # line 218 "lib/ll/lexer.rl" begin te = p+1 begin emit(:T_HEADER, ts, te) end end when 16 then # line 220 "lib/ll/lexer.rl" begin te = p+1 begin emit(:T_EQUALS, ts, te) end end when 14 then # line 221 "lib/ll/lexer.rl" begin te = p+1 begin emit(:T_COLON, ts, te) end end when 15 then # line 222 "lib/ll/lexer.rl" begin te = p+1 begin emit(:T_SEMICOLON, ts, te) end end when 20 then # line 223 "lib/ll/lexer.rl" begin te = p+1 begin emit(:T_PIPE, ts, te) end end when 13 then # line 225 "lib/ll/lexer.rl" begin te = p+1 begin emit(:T_PLUS, ts, te) end end when 12 then # line 226 "lib/ll/lexer.rl" begin te = p+1 begin emit(:T_STAR, ts, te) end end when 17 then # line 227 "lib/ll/lexer.rl" begin te = p+1 begin emit(:T_QUESTION, ts, te) end end when 10 then # line 228 "lib/ll/lexer.rl" begin te = p+1 begin emit(:T_LPAREN, ts, te) end end when 11 then # line 229 "lib/ll/lexer.rl" begin te = p+1 begin emit(:T_RPAREN, ts, te) end end when 19 then # line 231 "lib/ll/lexer.rl" begin te = p+1 begin mark = ts + 1 brace_count = 1 start_line = @line advance_column cs = 25; end end when 22 then # line 213 "lib/ll/lexer.rl" begin te = p p = p - 1; end when 21 then # line 1 "NONE" begin case act when 16 then begin begin p = ((te))-1; end emit(:T_EPSILON, ts, te) end when 23 then begin begin p = ((te))-1; end emit(:T_IDENT, ts, te) end end end when 24 then # line 148 "lib/ll/lexer.rl" begin advance_line @column = 1 end # line 186 "lib/ll/lexer.rl" begin te = p+1 end when 9 then # line 148 "lib/ll/lexer.rl" begin advance_line @column = 1 end # line 210 "lib/ll/lexer.rl" begin te = p+1 end when 8 then # line 154 "lib/ll/lexer.rl" begin advance_column end # line 211 "lib/ll/lexer.rl" begin te = p+1 end when 18 then # line 1 "NONE" begin te = p+1 end # line 224 "lib/ll/lexer.rl" begin act = 16; end when 7 then # line 1 "NONE" begin te = p+1 end # line 168 "lib/ll/lexer.rl" begin act = 23; end # line 507 "lib/ll/lexer.rb" end end end if _goto_level <= _again case _ll_lexer_to_state_actions[cs] when 5 then # line 1 "NONE" begin ts = nil; end # line 517 "lib/ll/lexer.rb" end if cs == 0 _goto_level = _out next end p += 1 if p != pe _goto_level = _resume next end end if _goto_level <= _test_eof if p == eof if _ll_lexer_eof_trans[cs] > 0 _trans = _ll_lexer_eof_trans[cs] - 1; _goto_level = _eof_trans next; end end end if _goto_level <= _out break end end end # line 81 "lib/ll/lexer.rl" # % fix highlight ensure reset end
Gathers all the tokens for the input and returns them as an Array.
@see [#advance] @return [Array]
# File lib/ll/lexer.rb, line 215 def lex tokens = [] advance do |token| tokens << token end return tokens end
Resets the internal state of the lexer.
# File lib/ll/lexer.rb, line 228 def reset @block = nil @line = 1 @column = 1 end
Private Instance Methods
Yields a new token to the supplied block.
@param [Symbol] type The token type. @param [String] value The token value. @param [Fixnum] line
@yieldparam [LL::Token] token
# File lib/ll/lexer.rb, line 592 def add_token(type, value, line = @line) source_line = SourceLine.new(@data, line, @column, @file) @column += value.length @block.call(Token.new(type, value, source_line)) end
# File lib/ll/lexer.rb, line 603 def advance_column @column += 1 end
# File lib/ll/lexer.rb, line 599 def advance_line @line += 1 end
Emits a token of which the value is based on the supplied start/stop position.
@param [Symbol] type The token type. @param [Fixnum] start @param [Fixnum] stop @param [Fixnum] line
@see [#text] @see [#add_token]
# File lib/ll/lexer.rb, line 566 def emit(type, start, stop, line = @line) value = slice_input(start, stop) add_token(type, value, line) end
Returns the text between the specified start and stop position.
@param [Fixnum] start @param [Fixnum] stop @return [String]
# File lib/ll/lexer.rb, line 579 def slice_input(start, stop) return @data.byteslice(start, stop - start) end