class Koara::TokenManager

Constants

ASTERISK
BACKSLASH
BACKTICK
CHAR_SEQUENCE
COLON
DASH
DIGITS
DOT
EOF
EOL
EQ
ESCAPED_CHAR
GT
IMAGE_LABEL
LBRACK
LPAREN
LT
RBRACK
RPAREN
SPACE
TAB
UNDERSCORE

Public Class Methods

new(stream) click to toggle source
# File lib/koara/token_manager.rb, line 31
def initialize(stream)
  @jj_rounds = Array.new(8, 0)
  @jj_state_set = Array.new(16, 0)
  @jj_next_states = [2, 3, 5]
  @cs = stream
  @round = 0
end

Public Instance Methods

check_n_add(state) click to toggle source
# File lib/koara/token_manager.rb, line 313
def check_n_add(state)
  if @jj_rounds[state] != @round
    @jj_state_set[@jj_new_state_cnt] = state
    @jj_new_state_cnt += 1
    @jj_rounds[state] = @round
  end
end
check_n_add_states(start, ending) click to toggle source
# File lib/koara/token_manager.rb, line 305
def check_n_add_states(start, ending)
  loop do
    check_n_add(@jj_next_states[start])
    break if start == ending
    start += 1
  end
end
fill_token() click to toggle source
# File lib/koara/token_manager.rb, line 66
def fill_token
  Token.new(@matched_kind, @cs.begin_line, @cs.begin_column, @cs.end_line, @cs.end_column, @cs.image)
end
get_next_token() click to toggle source
# File lib/koara/token_manager.rb, line 39
def get_next_token
  #   begin
  while true
    begin
      @cur_char = @cs.begin_token
    rescue
      @matched_kind = 0
      @matched_pos = -1
      return fill_token
    end

    @matched_kind = 2147483647
    @matched_pos = 0
    cur_pos = move_string_literal_dfa0

    if @matched_kind != 2147483647
      if (@matched_pos + 1) < cur_pos
        @cs.backup(cur_pos - @matched_pos - 1)
      end
      return fill_token
    end
  end
  #    rescue => err
  #      return nil
  #    end
end
move_nfa(start_state, cur_pos) click to toggle source
# File lib/koara/token_manager.rb, line 174
def move_nfa(start_state, cur_pos)
  starts_at = 0
  @jj_new_state_cnt = 8
  i = 1
  @jj_state_set[0] = start_state
  kind = 0x7fffffff

  while true
    if (@round += 1) == 0x7fffffff
      @round = 0x80000001
    end
    if @cur_char.ord < 64
      l = 1 << @cur_char.ord
      loop do
        i-=1

        case @jj_state_set[i]
          when 6
            if (0x880098feffffd9ff & l) != 0
              kind = 4 if kind > 4
              check_n_add(0)
            elsif ((0x3ff000000000000 & l) != 0)
              kind = 7 if kind > 7
              check_n_add(1)
            elsif (0x2400 & l) != 0
              kind = 9 if kind > 9
            elsif (0x100000200 & l) != 0
              check_n_add_states(0, 2)
            end
            if @cur_char.ord == 13
              @jj_state_set[@jj_new_state_cnt+=1] = 4
            end
          when 8
            if ((0x2400 & l) != 0)
              kind = 9 if kind > 9
            elsif (0x100000200 & l) != 0
              check_n_add_states(0, 2)
            end
            if @cur_char.ord == 13
              @jj_state_set[@jj_new_state_cnt+=1] = 4
            end
          when 0
            if (0x880098feffffd9ff & l) != 0
              kind = 4
              check_n_add(0)
            end
          when 1
            if (0x3ff000000000000 & l) != 0
              kind = 7 if kind > 7
              check_n_add(1)
            end
          when 2
            if (0x100000200 & l) != 0
              check_n_add_states(0, 2)
            end
          when 3
            if (0x2400 & l) != 0 && kind > 9
              kind = 9
            end
          when 4
            if @cur_char.ord == 10 && kind > 9
              kind = 9
            end
          when 5
            if @cur_char.ord == 13
              @jj_state_set[@jj_new_state_cnt+=1] = 4
            end
          when 7
            if (0x77ff670000000000 & l) != 0 && kind > 11
              kind = 11
            end
        end
        break if (i == starts_at)
      end
    elsif @cur_char.ord < 128
      l = 1 << (@cur_char.ord & 077)
      loop do
        i -= 1
        case @jj_state_set[i]
          when 6
            if l != 0
              kind = 4 if kind > 4
              check_n_add(0)
            elsif @cur_char == 92
              @jj_state_set[@jj_new_state_cnt+=1] = 7
            end
          when 0
            if (0xfffffffe47ffffff & l) != 0
              kind = 4
              check_n_add(0)
            end
          when 7
            if (0x1b8000000 & l) != 0 && kind > 11
              kind = 11
            end
        end
        break if (i == starts_at)
      end
    else
      loop do
        i-=1
        case @jj_state_set[i]
          when 6
            kind = 4 if kind > 4
            check_n_add(0)
          when 0
            kind = 4 if kind > 4
            check_n_add(0)
        end
        break if i == starts_at
      end
    end
    if kind != 0x7fffffff
      @matched_kind = kind
      @matched_pos = cur_pos
      kind = 0x7fffffff
    end
    cur_pos += 1

    if (i = @jj_new_state_cnt) == (starts_at = 8 - (@jj_new_state_cnt = starts_at))
      return cur_pos
    end

    begin
      @cur_char = @cs.read_char
    rescue => error
      return cur_pos
    end
  end
end
move_string_literal_dfa0() click to toggle source
# File lib/koara/token_manager.rb, line 70
def move_string_literal_dfa0
  case @cur_char.ord
    when 9
      return start_nfa_with_states(0, TAB, 8)
    when 32
      return start_nfa_with_states(0, SPACE, 8)
    when 40
      return stop_at_pos(0, LPAREN)
    when 41
      return stop_at_pos(0, RPAREN)
    when 42
      return stop_at_pos(0, ASTERISK)
    when 45
      return stop_at_pos(0, DASH)
    when 46
      return stop_at_pos(0, DOT)
    when 58
      return stop_at_pos(0, COLON)
    when 60
      return stop_at_pos(0, LT)
    when 61
      return stop_at_pos(0, EQ)
    when 62
      return stop_at_pos(0, GT)
    when 73
      return move_string_literal_dfa1(0x2000)
    when 91
      return stop_at_pos(0, LBRACK)
    when 92
      return start_nfa_with_states(0, BACKSLASH, 7)
    when 93
      return stop_at_pos(0, RBRACK)
    when 95
      return stop_at_pos(0, UNDERSCORE)
    when 96
      return stop_at_pos(0, BACKTICK)
    when 105
      return move_string_literal_dfa1(0x2000)
    else
      return move_nfa(6, 0)
  end
end
move_string_literal_dfa1(active) click to toggle source
# File lib/koara/token_manager.rb, line 130
def move_string_literal_dfa1(active)
  @cur_char = @cs.read_char
  if @cur_char.ord == 77 || @cur_char.ord == 109
    return move_string_literal_dfa2(active, 0x2000)
  end
  start_nfa(0, active)
end
move_string_literal_dfa2(old, active) click to toggle source
# File lib/koara/token_manager.rb, line 138
def move_string_literal_dfa2(old, active)
  @cur_char = @cs.read_char
  if @cur_char.ord == 65 || @cur_char.ord == 97
    return move_string_literal_dfa3(active, 0x2000)
  end
  start_nfa(1, active)
end
move_string_literal_dfa3(old, active) click to toggle source
# File lib/koara/token_manager.rb, line 146
def move_string_literal_dfa3(old, active)
  @cur_char = @cs.read_char
  if @cur_char.ord == 71 || @cur_char.ord == 103
    return move_string_literal_dfa4(active, 0x2000)
  end
  start_nfa(2, active)
end
move_string_literal_dfa4(old, active) click to toggle source
# File lib/koara/token_manager.rb, line 154
def move_string_literal_dfa4(old, active)
  @cur_char = @cs.read_char
  if @cur_char.ord == 69 || @cur_char.ord == 101
    return move_string_literal_dfa5(active, 0x2000)
  end
  start_nfa(3, active)
end
move_string_literal_dfa5(old, active) click to toggle source
# File lib/koara/token_manager.rb, line 162
def move_string_literal_dfa5(old, active)
  @cur_char = @cs.read_char()
  if @cur_char.ord == 58 && ((active & 0x2000) != 0)
    return stop_at_pos(5, 13)
  end
  start_nfa(4, active)
end
start_nfa(pos, active) click to toggle source
# File lib/koara/token_manager.rb, line 170
def start_nfa(pos, active)
  move_nfa(stop_string_literal_dfa(pos, active), pos + 1)
end
start_nfa_with_states(pos, kind, state) click to toggle source
# File lib/koara/token_manager.rb, line 113
def start_nfa_with_states(pos, kind, state)
  @matched_kind = kind
  @matched_pos = pos
  begin
    @cur_char = @cs.read_char
  rescue
    return pos + 1
  end
  move_nfa(state, pos + 1)
end
stop_at_pos(pos, kind) click to toggle source
# File lib/koara/token_manager.rb, line 124
def stop_at_pos(pos, kind)
  @matched_kind = kind
  @matched_pos = pos
  pos + 1
end
stop_string_literal_dfa(pos, active) click to toggle source
# File lib/koara/token_manager.rb, line 321
def stop_string_literal_dfa(pos, active)
  if pos == 0
    if (active & 0x2000) != 0
      @matched_kind = 4
      return 0
    elsif (active & 0x180000) != 0
      return 8
    elsif (active & 0x4) != 0
      return 7
    end
  elsif pos == 1 && (active & 0x2000) != 0
    @matched_kind = 4
    @matched_pos = 1
    return 0
  elsif pos == 2 && (active & 0x2000) != 0
    @matched_kind = 4
    @matched_pos = 2
    return 0
  elsif pos == 3 && (active & 0x2000) != 0
    @matched_kind = 4
    @matched_pos = 3
    return 0
  elsif pos == 4 && (active & 0x2000) != 0
    @matched_kind = 4
    @matched_pos = 4
    return 0
  end
  -1
end