class ABNF::Parser::Compiler::Tokenizer

Constants

C_NL
C_WSP

Attributes

abnf[R]

Public Class Methods

build(abnf) click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 14
def self.build abnf
  new abnf.dup
end
call(abnf) click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 18
def self.call abnf
  instance = build abnf
  instance.()
end
new(abnf) click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 10
def initialize abnf
  @abnf = abnf
end

Public Instance Methods

apply(regexp) click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 23
def apply regexp
  match_data = regexp.match abnf
  return if match_data.nil? or not match_data.pre_match.empty?
  abnf.slice! 0, match_data.to_s.size
  match_data
end
call(&block) click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 30
def call &block
  tokens = []

  each do |token|
    tokens << token
  end

  tokens
end
each(&block) click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 40
def each &block
  until abnf.empty?
    length = abnf.bytesize

    token = self.next
    block.(token) if block

    fail "Parse error" unless abnf.bytesize < length
  end
end
next() click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 51
def next
  possible_tokens.each do |type, regexp|
    match_data = apply regexp
    next unless match_data

    token = Token.new type, match_data.to_s
    return token
  end

  nil
end
possible_tokens() click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 63
def possible_tokens
  @@possible_tokens ||= {
    'prose-val'.freeze         => %r{\A<[\x20-\x3D\x3F-\x7E]*>}n,
    'hex-val-RANGE'.freeze     => %r{\A%x[[:xdigit:]]+-[[:xdigit:]]+}n,
    'hex-val-SEQUENCE'.freeze  => %r{\A%x[[:xdigit:]]+(?:\.[[:xdigit:]]+)*}n,
    'dec-val-RANGE'.freeze     => %r{\A%d[[:digit:]]+-[[:digit:]]+}n,
    'dec-val-SEQUENCE'.freeze  => %r{\A%d[[:digit:]]+(?:\.[[:digit:]]+)*}n,
    'bin-val-RANGE'.freeze     => %r{\A%b[01]+-[01]+}n,
    'bin-val-SEQUENCE'.freeze  => %r{\A%b[01]+(?:\.[01]+)*}n,
    'char-val'.freeze          => %r{\A"[\x20-\x21\x23-\x7E]*"}n,
    'option-START'.freeze      => %r{\A\[#{C_WSP}*}n,
    'option-STOP'.freeze       => %r{\A#{C_WSP}*\]}n,
    'group-START'.freeze       => %r{\A\(#{C_WSP}*}n,
    'group-STOP'.freeze        => %r{\A#{C_WSP}*\)}n,
    'repeat-RANGE'.freeze      => %r{\A[[:digit:]]*\*[[:digit:]]*}n,
    'repeat-EXACT'.freeze      => %r{\A[[:digit:]]+}n,
    'alternation-SLASH'.freeze => %r{\A#{C_WSP}*/#{C_WSP}*}n,
    'defined-as'.freeze        => %r{\A#{C_WSP}*=/?#{C_WSP}*}n,
    'rulename'.freeze          => %r{\A[[:alpha:]][-[:alnum:]]*}n,
    'c-wsp'.freeze             => %r{\A(?:(?:#{C_NL})?[[:blank:]])+}n,
    'c-nl'.freeze              => %r{\A#{C_NL}}n,
  }
end
to_enum() click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 87
def to_enum
  Enumerator::Lazy.new self do |yielder, token|
    yielder << token
  end
end