class SQLPP::Tokenizer
Constants
- KEYWORDS
- KEYWORDS_REGEX
Public Class Methods
new(string)
click to toggle source
# File lib/sqlpp/tokenizer.rb, line 55 def initialize(string) @scanner = StringScanner.new(string) @buffer = [] end
Public Instance Methods
_scan()
click to toggle source
# File lib/sqlpp/tokenizer.rb, line 77 def _scan pos = @scanner.pos if @scanner.eos? Token.new(:eof, nil, pos) elsif (key = @scanner.scan(KEYWORDS_REGEX)) Token.new(:key, key.downcase.to_sym, pos) elsif (num = @scanner.scan(/\d+(?:\.\d+)?/)) Token.new(:lit, num, pos) elsif (id = @scanner.scan(/\w+/)) Token.new(:id, id, pos) elsif (punct = @scanner.scan(/<=|<>|!=|>=|::/)) Token.new(:punct, punct, pos) elsif (punct = @scanner.scan(/[<>=\(\).*,\/+\-\[\]]/)) Token.new(:punct, punct, pos) elsif (delim = @scanner.scan(/["`]/)) contents = _scan_to_delim(delim, pos) Token.new(:id, "#{delim}#{contents}#{delim}", pos) elsif @scanner.scan(/'/) contents = _scan_to_delim("'", pos) Token.new(:lit, "'#{contents}'", pos) elsif (space = @scanner.scan(/\s+/)) Token.new(:space, space, pos) else raise UnexpectedCharacter, @scanner.rest end end
_scan_to_delim(delim, pos)
click to toggle source
# File lib/sqlpp/tokenizer.rb, line 105 def _scan_to_delim(delim, pos) escape, if_peek = case delim when '"', '`' then ["\\", nil] when "'" then ["'", "'"] end string = "" loop do ch = @scanner.getch if ch == escape && (if_peek.nil? || @scanner.peek(1) == if_peek) ch << @scanner.getch end case ch when nil then raise EOFError, "end of input reached in string started at #{pos} with #{delim.inspect}" when delim then return string else string << ch end end end
next()
click to toggle source
# File lib/sqlpp/tokenizer.rb, line 60 def next if @buffer.any? @buffer.pop else _scan end end
peek()
click to toggle source
# File lib/sqlpp/tokenizer.rb, line 68 def peek push(self.next) end
push(token)
click to toggle source
# File lib/sqlpp/tokenizer.rb, line 72 def push(token) @buffer.push(token) token end