class LL::Lexer

Ragel lexer for LL grammar files.

Attributes

_ll_lexer_eof_trans[RW]
_ll_lexer_from_state_actions[RW]
_ll_lexer_index_offsets[RW]
_ll_lexer_indicies[RW]
_ll_lexer_key_spans[RW]
_ll_lexer_to_state_actions[RW]
_ll_lexer_trans_actions[RW]
_ll_lexer_trans_keys[RW]
_ll_lexer_trans_targs[RW]
ll_lexer_en_main[RW]
ll_lexer_en_ruby_body[RW]
ll_lexer_error[RW]
ll_lexer_first_final[RW]
ll_lexer_start[RW]

Public Class Methods

new(data, file = SourceLine::DEFAULT_FILE) click to toggle source

@param [String] data The data to lex. @param [String] file The name of the input file.

# File lib/ll/lexer.rb, line 202
def initialize(data, file = SourceLine::DEFAULT_FILE)
  @data = data
  @file = file

  reset
end

Public Instance Methods

advance(&block) click to toggle source

Advances through the input and generates the corresponding tokens. Each token is yielded to the supplied block.

@see [#add_token]

# File lib/ll/lexer.rb, line 240
    def advance(&block)
      @block = block

      data = @data # saves ivar lookups while lexing.
      ts   = nil
      te   = nil
      cs   = self.class.ll_lexer_start
      act  = 0
      eof  = @data.bytesize
      p    = 0
      pe   = eof

      mark        = 0
      brace_count = 0
      start_line  = 0

      _ll_lexer_eof_trans          = self.class.send(:_ll_lexer_eof_trans)
      _ll_lexer_from_state_actions = self.class.send(:_ll_lexer_from_state_actions)
      _ll_lexer_index_offsets      = self.class.send(:_ll_lexer_index_offsets)
      _ll_lexer_indicies           = self.class.send(:_ll_lexer_indicies)
      _ll_lexer_key_spans          = self.class.send(:_ll_lexer_key_spans)
      _ll_lexer_to_state_actions   = self.class.send(:_ll_lexer_to_state_actions)
      _ll_lexer_trans_actions      = self.class.send(:_ll_lexer_trans_actions)
      _ll_lexer_trans_keys         = self.class.send(:_ll_lexer_trans_keys)
      _ll_lexer_trans_targs        = self.class.send(:_ll_lexer_trans_targs)

      
# line 268 "lib/ll/lexer.rb"
begin
        testEof = false
        _slen, _trans, _keys, _inds, _acts, _nacts = nil
        _goto_level = 0
        _resume = 10
        _eof_trans = 15
        _again = 20
        _test_eof = 30
        _out = 40
        while true
        if _goto_level <= 0
        if p == pe
                _goto_level = _test_eof
                next
        end
        if cs == 0
                _goto_level = _out
                next
        end
        end
        if _goto_level <= _resume
        case _ll_lexer_from_state_actions[cs] 
        when 6 then
# line 1 "NONE"
                begin
ts = p
                end
# line 296 "lib/ll/lexer.rb"
        end
        _keys = cs << 1
        _inds = _ll_lexer_index_offsets[cs]
        _slen = _ll_lexer_key_spans[cs]
        _wide = ( (data.getbyte(p) || 0))
        _trans = if (   _slen > 0 && 
                        _ll_lexer_trans_keys[_keys] <= _wide && 
                        _wide <= _ll_lexer_trans_keys[_keys + 1] 
                    ) then
                        _ll_lexer_indicies[ _inds + _wide - _ll_lexer_trans_keys[_keys] ] 
                 else 
                        _ll_lexer_indicies[ _inds + _slen ]
                 end
        end
        if _goto_level <= _eof_trans
        cs = _ll_lexer_trans_targs[_trans]
        if _ll_lexer_trans_actions[_trans] != 0
        case _ll_lexer_trans_actions[_trans]
        when 25 then
# line 188 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  brace_count += 1  end
                end
        when 26 then
# line 190 "lib/ll/lexer.rl"
                begin
te = p+1
 begin 
          if brace_count == 1
            emit(:T_RUBY, mark, ts, start_line)

            mark        = 0
            brace_count = 0
            start_line  = 0

            advance_column

            cs = 22;
          else
            brace_count -= 1
          end
         end
                end
        when 23 then
# line 206 "lib/ll/lexer.rl"
                begin
te = p+1
                end
        when 3 then
# line 215 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  emit(:T_NAME, ts, te)  end
                end
        when 4 then
# line 216 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  emit(:T_TERMINALS, ts, te)  end
                end
        when 2 then
# line 217 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  emit(:T_INNER, ts, te)  end
                end
        when 1 then
# line 218 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  emit(:T_HEADER, ts, te)  end
                end
        when 16 then
# line 220 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  emit(:T_EQUALS, ts, te)  end
                end
        when 14 then
# line 221 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  emit(:T_COLON, ts, te)  end
                end
        when 15 then
# line 222 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  emit(:T_SEMICOLON, ts, te)  end
                end
        when 20 then
# line 223 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  emit(:T_PIPE, ts, te)  end
                end
        when 13 then
# line 225 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  emit(:T_PLUS, ts, te)  end
                end
        when 12 then
# line 226 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  emit(:T_STAR, ts, te)  end
                end
        when 17 then
# line 227 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  emit(:T_QUESTION, ts, te)  end
                end
        when 10 then
# line 228 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  emit(:T_LPAREN, ts, te)  end
                end
        when 11 then
# line 229 "lib/ll/lexer.rl"
                begin
te = p+1
 begin  emit(:T_RPAREN, ts, te)  end
                end
        when 19 then
# line 231 "lib/ll/lexer.rl"
                begin
te = p+1
 begin 
          mark        = ts + 1
          brace_count = 1
          start_line  = @line

          advance_column

          cs = 25;
         end
                end
        when 22 then
# line 213 "lib/ll/lexer.rl"
                begin
te = p
p = p - 1;              end
        when 21 then
# line 1 "NONE"
                begin
        case act
        when 16 then
        begin begin p = ((te))-1; end
 emit(:T_EPSILON, ts, te) end
        when 23 then
        begin begin p = ((te))-1; end

        emit(:T_IDENT, ts, te)
      end
end 
                        end
        when 24 then
# line 148 "lib/ll/lexer.rl"
                begin

        advance_line

        @column = 1
                end
# line 186 "lib/ll/lexer.rl"
                begin
te = p+1
                end
        when 9 then
# line 148 "lib/ll/lexer.rl"
                begin

        advance_line

        @column = 1
                end
# line 210 "lib/ll/lexer.rl"
                begin
te = p+1
                end
        when 8 then
# line 154 "lib/ll/lexer.rl"
                begin

        advance_column
                end
# line 211 "lib/ll/lexer.rl"
                begin
te = p+1
                end
        when 18 then
# line 1 "NONE"
                begin
te = p+1
                end
# line 224 "lib/ll/lexer.rl"
                begin
act = 16;               end
        when 7 then
# line 1 "NONE"
                begin
te = p+1
                end
# line 168 "lib/ll/lexer.rl"
                begin
act = 23;               end
# line 507 "lib/ll/lexer.rb"
        end
        end
        end
        if _goto_level <= _again
        case _ll_lexer_to_state_actions[cs] 
        when 5 then
# line 1 "NONE"
                begin
ts = nil;               end
# line 517 "lib/ll/lexer.rb"
        end

        if cs == 0
                _goto_level = _out
                next
        end
        p += 1
        if p != pe
                _goto_level = _resume
                next
        end
        end
        if _goto_level <= _test_eof
        if p == eof
        if _ll_lexer_eof_trans[cs] > 0
                _trans = _ll_lexer_eof_trans[cs] - 1;
                _goto_level = _eof_trans
                next;
        end
        end

        end
        if _goto_level <= _out
                break
        end
end
        end

# line 81 "lib/ll/lexer.rl"

      # % fix highlight
    ensure
      reset
    end
lex() click to toggle source

Gathers all the tokens for the input and returns them as an Array.

@see [#advance] @return [Array]

# File lib/ll/lexer.rb, line 215
def lex
  tokens = []

  advance do |token|
    tokens << token
  end

  return tokens
end
reset() click to toggle source

Resets the internal state of the lexer.

# File lib/ll/lexer.rb, line 228
def reset
  @block  = nil
  @line   = 1
  @column = 1
end

Private Instance Methods

add_token(type, value, line = @line) click to toggle source

Yields a new token to the supplied block.

@param [Symbol] type The token type. @param [String] value The token value. @param [Fixnum] line

@yieldparam [LL::Token] token

# File lib/ll/lexer.rb, line 592
def add_token(type, value, line = @line)
  source_line = SourceLine.new(@data, line, @column, @file)
  @column    += value.length

  @block.call(Token.new(type, value, source_line))
end
advance_column() click to toggle source
# File lib/ll/lexer.rb, line 603
def advance_column
  @column += 1
end
advance_line() click to toggle source
# File lib/ll/lexer.rb, line 599
def advance_line
  @line += 1
end
emit(type, start, stop, line = @line) click to toggle source

Emits a token of which the value is based on the supplied start/stop position.

@param [Symbol] type The token type. @param [Fixnum] start @param [Fixnum] stop @param [Fixnum] line

@see [#text] @see [#add_token]

# File lib/ll/lexer.rb, line 566
def emit(type, start, stop, line = @line)
  value = slice_input(start, stop)

  add_token(type, value, line)
end
slice_input(start, stop) click to toggle source

Returns the text between the specified start and stop position.

@param [Fixnum] start @param [Fixnum] stop @return [String]

# File lib/ll/lexer.rb, line 579
def slice_input(start, stop)
  return @data.byteslice(start, stop - start)
end