class ABNF::Parser::Compiler::Tokenizer
Constants
- C_NL
- C_WSP
Attributes
abnf[R]
Public Class Methods
build(abnf)
click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 14 def self.build abnf new abnf.dup end
call(abnf)
click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 18 def self.call abnf instance = build abnf instance.() end
new(abnf)
click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 10 def initialize abnf @abnf = abnf end
Public Instance Methods
apply(regexp)
click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 23 def apply regexp match_data = regexp.match abnf return if match_data.nil? or not match_data.pre_match.empty? abnf.slice! 0, match_data.to_s.size match_data end
call(&block)
click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 30 def call &block tokens = [] each do |token| tokens << token end tokens end
each(&block)
click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 40 def each &block until abnf.empty? length = abnf.bytesize token = self.next block.(token) if block fail "Parse error" unless abnf.bytesize < length end end
next()
click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 51 def next possible_tokens.each do |type, regexp| match_data = apply regexp next unless match_data token = Token.new type, match_data.to_s return token end nil end
possible_tokens()
click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 63 def possible_tokens @@possible_tokens ||= { 'prose-val'.freeze => %r{\A<[\x20-\x3D\x3F-\x7E]*>}n, 'hex-val-RANGE'.freeze => %r{\A%x[[:xdigit:]]+-[[:xdigit:]]+}n, 'hex-val-SEQUENCE'.freeze => %r{\A%x[[:xdigit:]]+(?:\.[[:xdigit:]]+)*}n, 'dec-val-RANGE'.freeze => %r{\A%d[[:digit:]]+-[[:digit:]]+}n, 'dec-val-SEQUENCE'.freeze => %r{\A%d[[:digit:]]+(?:\.[[:digit:]]+)*}n, 'bin-val-RANGE'.freeze => %r{\A%b[01]+-[01]+}n, 'bin-val-SEQUENCE'.freeze => %r{\A%b[01]+(?:\.[01]+)*}n, 'char-val'.freeze => %r{\A"[\x20-\x21\x23-\x7E]*"}n, 'option-START'.freeze => %r{\A\[#{C_WSP}*}n, 'option-STOP'.freeze => %r{\A#{C_WSP}*\]}n, 'group-START'.freeze => %r{\A\(#{C_WSP}*}n, 'group-STOP'.freeze => %r{\A#{C_WSP}*\)}n, 'repeat-RANGE'.freeze => %r{\A[[:digit:]]*\*[[:digit:]]*}n, 'repeat-EXACT'.freeze => %r{\A[[:digit:]]+}n, 'alternation-SLASH'.freeze => %r{\A#{C_WSP}*/#{C_WSP}*}n, 'defined-as'.freeze => %r{\A#{C_WSP}*=/?#{C_WSP}*}n, 'rulename'.freeze => %r{\A[[:alpha:]][-[:alnum:]]*}n, 'c-wsp'.freeze => %r{\A(?:(?:#{C_NL})?[[:blank:]])+}n, 'c-nl'.freeze => %r{\A#{C_NL}}n, } end
to_enum()
click to toggle source
# File lib/abnf/parser/compiler/tokenizer.rb, line 87 def to_enum Enumerator::Lazy.new self do |yielder, token| yielder << token end end