class SLF0::Tokenizer

Attributes

scanner[R]

Public Class Methods

tokenize(slf0) click to toggle source
# File lib/slf0/tokenizer.rb, line 11
def self.tokenize(slf0)
  new(slf0).tokenize
end

Private Class Methods

new(slf0) click to toggle source
# File lib/slf0/tokenizer.rb, line 6
def initialize(slf0)
  @scanner = StringScanner.new(slf0)
end

Public Instance Methods

scan_length(length) click to toggle source
# File lib/slf0/tokenizer.rb, line 55
def scan_length(length)
  string = scanner.string[scanner.pos, length]
  scanner.pos += length
  string
end
skip_header!() click to toggle source
# File lib/slf0/tokenizer.rb, line 20
def skip_header!
  raise 'missing header' unless scanner.skip(/SLF0/)
end
tokenize() click to toggle source
# File lib/slf0/tokenizer.rb, line 15
def tokenize
  skip_header!
  tokenize_body!
end
tokenize_body!() click to toggle source
# File lib/slf0/tokenizer.rb, line 24
def tokenize_body!
  body = []
  until scanner.eos?
    object = tokenize_double_field || tokenize_field || tokenize_object_list_nil
    raise "malformed no object: #{scanner.rest.inspect}\n\nafter: #{body.inspect}" unless object

    body << object
  end
  body
end
tokenize_double_field() click to toggle source
# File lib/slf0/tokenizer.rb, line 61
def tokenize_double_field
  return unless (hex = scanner.scan(/\h*\^/)&.chop)

  double = [hex.to_i(16)].pack('Q<').unpack1('G')
  SLF0::Token::Double.new double
end
tokenize_field() click to toggle source
# File lib/slf0/tokenizer.rb, line 35
def tokenize_field
  raise 'no int found' unless (int = scanner.scan(/\d+/).to_i)

  case scanner.get_byte
  when '#'
    SLF0::Token::Int.new int
  when '%'
    SLF0::Token::ClassName.new scan_length(int).freeze
  when '@'
    SLF0::Token::ClassNameRef.new int
  when '"'
    SLF0::Token::String.new scan_length(int).tr("\r", "\n").freeze
  when '('
    SLF0::Token::ObjectList.new int
  else
    scanner.unscan
    nil
  end
end
tokenize_object_list_nil() click to toggle source
# File lib/slf0/tokenizer.rb, line 68
def tokenize_object_list_nil
  return unless scanner.skip(/-/)

  SLF0::Token::ObjectListNil.new nil
end