class SLF0::Tokenizer
Attributes
scanner[R]
Public Class Methods
tokenize(slf0)
click to toggle source
# File lib/slf0/tokenizer.rb, line 11 def self.tokenize(slf0) new(slf0).tokenize end
Private Class Methods
new(slf0)
click to toggle source
# File lib/slf0/tokenizer.rb, line 6 def initialize(slf0) @scanner = StringScanner.new(slf0) end
Public Instance Methods
scan_length(length)
click to toggle source
# File lib/slf0/tokenizer.rb, line 55 def scan_length(length) string = scanner.string[scanner.pos, length] scanner.pos += length string end
skip_header!()
click to toggle source
# File lib/slf0/tokenizer.rb, line 20 def skip_header! raise 'missing header' unless scanner.skip(/SLF0/) end
tokenize()
click to toggle source
# File lib/slf0/tokenizer.rb, line 15 def tokenize skip_header! tokenize_body! end
tokenize_body!()
click to toggle source
# File lib/slf0/tokenizer.rb, line 24 def tokenize_body! body = [] until scanner.eos? object = tokenize_double_field || tokenize_field || tokenize_object_list_nil raise "malformed no object: #{scanner.rest.inspect}\n\nafter: #{body.inspect}" unless object body << object end body end
tokenize_double_field()
click to toggle source
# File lib/slf0/tokenizer.rb, line 61 def tokenize_double_field return unless (hex = scanner.scan(/\h*\^/)&.chop) double = [hex.to_i(16)].pack('Q<').unpack1('G') SLF0::Token::Double.new double end
tokenize_field()
click to toggle source
# File lib/slf0/tokenizer.rb, line 35 def tokenize_field raise 'no int found' unless (int = scanner.scan(/\d+/).to_i) case scanner.get_byte when '#' SLF0::Token::Int.new int when '%' SLF0::Token::ClassName.new scan_length(int).freeze when '@' SLF0::Token::ClassNameRef.new int when '"' SLF0::Token::String.new scan_length(int).tr("\r", "\n").freeze when '(' SLF0::Token::ObjectList.new int else scanner.unscan nil end end
tokenize_object_list_nil()
click to toggle source
# File lib/slf0/tokenizer.rb, line 68 def tokenize_object_list_nil return unless scanner.skip(/-/) SLF0::Token::ObjectListNil.new nil end