class SQLPP::Tokenizer

Constants

KEYWORDS
KEYWORDS_REGEX

Public Class Methods

new(string) click to toggle source
# File lib/sqlpp/tokenizer.rb, line 55
def initialize(string)
  @scanner = StringScanner.new(string)
  @buffer = []
end

Public Instance Methods

_scan() click to toggle source
# File lib/sqlpp/tokenizer.rb, line 77
def _scan
  pos = @scanner.pos

  if @scanner.eos?
    Token.new(:eof, nil, pos)
  elsif (key = @scanner.scan(KEYWORDS_REGEX))
    Token.new(:key, key.downcase.to_sym, pos)
  elsif (num = @scanner.scan(/\d+(?:\.\d+)?/))
    Token.new(:lit, num, pos)
  elsif (id = @scanner.scan(/\w+/))
    Token.new(:id, id, pos)
  elsif (punct = @scanner.scan(/<=|<>|!=|>=|::/))
    Token.new(:punct, punct, pos)
  elsif (punct = @scanner.scan(/[<>=\(\).*,\/+\-\[\]]/))
    Token.new(:punct, punct, pos)
  elsif (delim = @scanner.scan(/["`]/))
    contents = _scan_to_delim(delim, pos)
    Token.new(:id, "#{delim}#{contents}#{delim}", pos)
  elsif @scanner.scan(/'/)
    contents = _scan_to_delim("'", pos)
    Token.new(:lit, "'#{contents}'", pos)
  elsif (space = @scanner.scan(/\s+/))
    Token.new(:space, space, pos)
  else
    raise UnexpectedCharacter, @scanner.rest
  end
end
_scan_to_delim(delim, pos) click to toggle source
# File lib/sqlpp/tokenizer.rb, line 105
def _scan_to_delim(delim, pos)
  escape, if_peek = case delim
    when '"', '`' then ["\\", nil]
    when "'" then ["'", "'"]
  end

  string = ""
  loop do
    ch = @scanner.getch

    if ch == escape && (if_peek.nil? || @scanner.peek(1) == if_peek)
      ch << @scanner.getch
    end

    case ch
    when nil then
      raise EOFError, "end of input reached in string started at #{pos} with #{delim.inspect}"
    when delim then
      return string
    else
      string << ch
    end
  end
end
next() click to toggle source
# File lib/sqlpp/tokenizer.rb, line 60
def next
  if @buffer.any?
    @buffer.pop
  else
    _scan
  end
end
peek() click to toggle source
# File lib/sqlpp/tokenizer.rb, line 68
def peek
  push(self.next)
end
push(token) click to toggle source
# File lib/sqlpp/tokenizer.rb, line 72
def push(token)
  @buffer.push(token)
  token
end