class Dentaku::TokenScanner

Constants

DATE_TIME_REGEXP

Attributes

case_sensitive[R]

Public Class Methods

access() click to toggle source
# File lib/dentaku/token_scanner.rb, line 143
def access
  names = { lbracket: '[', rbracket: ']' }.invert
  new(:access, '\[|\]', lambda { |raw| names[raw] })
end
array() click to toggle source
# File lib/dentaku/token_scanner.rb, line 138
def array
  names = { array_start: '{', array_end: '}', }.invert
  new(:array, '\{|\}|,', lambda { |raw| names[raw] })
end
available_scanners() click to toggle source
# File lib/dentaku/token_scanner.rb, line 35
def available_scanners
  [
    :null,
    :whitespace,
    :datetime, # before numeric so it can pick up timestamps
    :numeric,
    :hexadecimal,
    :double_quoted_string,
    :single_quoted_string,
    :negate,
    :combinator,
    :operator,
    :grouping,
    :array,
    :access,
    :case_statement,
    :comparator,
    :boolean,
    :function,
    :identifier,
    :quoted_identifier
  ]
end
boolean() click to toggle source
# File lib/dentaku/token_scanner.rb, line 167
def boolean
  new(:logical, '(true|false)\b', lambda { |raw| raw.strip.downcase == 'true' })
end
case_statement() click to toggle source
# File lib/dentaku/token_scanner.rb, line 148
def case_statement
  names = { open: 'case', close: 'end', then: 'then', when: 'when', else: 'else' }.invert
  new(:case, '(case|end|then|when|else)\b', lambda { |raw| names[raw.downcase] })
end
combinator() click to toggle source
# File lib/dentaku/token_scanner.rb, line 159
def combinator
  names = { and: '&&', or: '||' }.invert
  new(:combinator, '(and|or|&&|\|\|)\s', lambda { |raw|
    norm = raw.strip.downcase
    names.fetch(norm) { norm.to_sym }
  })
end
comparator() click to toggle source
# File lib/dentaku/token_scanner.rb, line 153
def comparator
  names = { le: '<=', ge: '>=', ne: '!=', lt: '<', gt: '>', eq: '=' }.invert
  alternate = { ne: '<>', eq: '==' }.invert
  new(:comparator, '<=|>=|!=|<>|<|>|==|=', lambda { |raw| names[raw] || alternate[raw] })
end
datetime() click to toggle source

NOTE: Convert to DateTime as Array(Time) returns the parts of the time for some reason

# File lib/dentaku/token_scanner.rb, line 93
def datetime
  new(:datetime, DATE_TIME_REGEXP, lambda { |raw| Time.parse(raw).to_datetime })
end
double_quoted_string() click to toggle source
# File lib/dentaku/token_scanner.rb, line 107
def double_quoted_string
  new(:string, '"[^"]*"', lambda { |raw| raw.gsub(/^"|"$/, '') })
end
function() click to toggle source
# File lib/dentaku/token_scanner.rb, line 171
def function
  new(:function, '\w+!?\s*\(', lambda do |raw|
    function_name = raw.gsub('(', '')
    [
      Token.new(:function, function_name.strip.downcase.to_sym, function_name),
      Token.new(:grouping, :open, '(')
    ]
  end)
end
grouping() click to toggle source
# File lib/dentaku/token_scanner.rb, line 133
def grouping
  names = { open: '(', close: ')', comma: ',' }.invert
  new(:grouping, '\(|\)|,', lambda { |raw| names[raw] })
end
hexadecimal() click to toggle source
# File lib/dentaku/token_scanner.rb, line 103
def hexadecimal
  new(:numeric, '(0x[0-9a-f]+)\b', lambda { |raw| raw[2..-1].to_i(16) })
end
identifier() click to toggle source
# File lib/dentaku/token_scanner.rb, line 181
def identifier
  new(:identifier, '[[[:word:]]\.]+\b', lambda { |raw| standardize_case(raw.strip) })
end
negate() click to toggle source
# File lib/dentaku/token_scanner.rb, line 115
def negate
  new(:operator, '-', lambda { |raw| :negate }, lambda { |last_token|
    last_token.nil?             ||
    last_token.is?(:operator)   ||
    last_token.is?(:comparator) ||
    last_token.is?(:combinator) ||
    last_token.value == :open   ||
    last_token.value == :comma
  })
end
new(category, regexp, converter = nil, condition = nil) click to toggle source
# File lib/dentaku/token_scanner.rb, line 12
def initialize(category, regexp, converter = nil, condition = nil)
  @category  = category
  @regexp    = %r{\A(#{ regexp })}i
  @converter = converter
  @condition = condition || ->(*) { true }
end
null() click to toggle source
# File lib/dentaku/token_scanner.rb, line 88
def null
  new(:null, 'null\b')
end
numeric() click to toggle source
# File lib/dentaku/token_scanner.rb, line 97
def numeric
  new(:numeric, '((?:\d+(\.\d+)?|\.\d+)(?:(e|E)(\+|-)?\d+)?)\b', lambda { |raw|
    raw =~ /(\.|e|E)/ ? BigDecimal(raw) : raw.to_i
  })
end
operator() click to toggle source
# File lib/dentaku/token_scanner.rb, line 126
def operator
  names = {
    pow: '^', add: '+', subtract: '-', multiply: '*', divide: '/', mod: '%', bitor: '|', bitand: '&', bitshiftleft: '<<', bitshiftright: '>>'
  }.invert
  new(:operator, '\^|\+|-|\*|\/|%|\||&|<<|>>', lambda { |raw| names[raw] })
end
quoted_identifier() click to toggle source
# File lib/dentaku/token_scanner.rb, line 185
def quoted_identifier
  new(:identifier, '`[^`]*`', lambda { |raw| raw.gsub(/^`|`$/, '') })
end
register_default_scanners() click to toggle source
# File lib/dentaku/token_scanner.rb, line 59
def register_default_scanners
  register_scanners(available_scanners)
end
register_scanner(id, scanner) click to toggle source
# File lib/dentaku/token_scanner.rb, line 69
def register_scanner(id, scanner)
  @scanners[id] = scanner
end
register_scanners(scanner_ids) click to toggle source
# File lib/dentaku/token_scanner.rb, line 63
def register_scanners(scanner_ids)
  @scanners = scanner_ids.each_with_object({}) do |id, scanners|
    scanners[id] = self.send(id)
  end
end
scanners(options = {}) click to toggle source
# File lib/dentaku/token_scanner.rb, line 77
def scanners(options = {})
  @case_sensitive = options.fetch(:case_sensitive, false)
  raw_date_literals = options.fetch(:raw_date_literals, true)

  @scanners.select { |k, _| raw_date_literals || k != :datetime }.values
end
scanners=(scanner_ids) click to toggle source
# File lib/dentaku/token_scanner.rb, line 73
def scanners=(scanner_ids)
  @scanners.select! { |k, v| scanner_ids.include?(k) }
end
single_quoted_string() click to toggle source
# File lib/dentaku/token_scanner.rb, line 111
def single_quoted_string
  new(:string, "'[^']*'", lambda { |raw| raw.gsub(/^'|'$/, '') })
end
whitespace() click to toggle source
# File lib/dentaku/token_scanner.rb, line 84
def whitespace
  new(:whitespace, '\s+')
end

Public Instance Methods

scan(string, last_token = nil) click to toggle source
# File lib/dentaku/token_scanner.rb, line 19
def scan(string, last_token = nil)
  if (m = @regexp.match(string)) && @condition.call(last_token)
    value = raw = m.to_s
    value = @converter.call(raw) if @converter

    return Array(value).map do |v|
      Token === v ? v : Token.new(@category, v, raw)
    end
  end

  false
end