class Arcamark::Lexer
Public Class Methods
new(source)
click to toggle source
# File lib/arcamark/lexer.rb, line 5 def initialize(source) @source = String(source).force_encoding('UTF-8').freeze @curr = @source[0].freeze @curr_index = 0 @curr_line = 1 @curr_column = 1 @tokens = [] raise 'Invalid character' if invalid? start_token end
Public Instance Methods
next_token()
click to toggle source
# File lib/arcamark/lexer.rb, line 20 def next_token case @state when :newline next_char while newline? finish_token :empty_line when :whitespace next_char while whitespace? finish_token :whitespace when :text next_char while text? finish_token :text when :funcall next_char while funcall? finish_token :funcall when :operator case @curr when '{' next_char finish_token :brace_open when '}' next_char finish_token :brace_close else raise 'Invalid state' end end end
Private Instance Methods
eof?()
click to toggle source
# File lib/arcamark/lexer.rb, line 50 def eof? @curr.nil? end
finish_token(type)
click to toggle source
# File lib/arcamark/lexer.rb, line 123 def finish_token(type) token = Token.new type, @value, @value_index, @value_line, @value_column @tokens << token start_token @tokens.last end
funcall?()
click to toggle source
# File lib/arcamark/lexer.rb, line 74 def funcall? !eof? && CharType.funcall?(@curr) end
funcall_start?()
click to toggle source
# File lib/arcamark/lexer.rb, line 70 def funcall_start? !eof? && CharType.funcall_start?(@curr) end
invalid?()
click to toggle source
# File lib/arcamark/lexer.rb, line 54 def invalid? !eof? && CharType.invalid?(@curr) end
newline?()
click to toggle source
# File lib/arcamark/lexer.rb, line 58 def newline? !eof? && CharType.newline?(@curr) end
next_char()
click to toggle source
# File lib/arcamark/lexer.rb, line 105 def next_char return if eof? @value += @curr if newline? @curr_line += 1 @curr_column = 1 else @curr_column += 1 end @curr_index += 1 @curr = @source[@curr_index].freeze raise 'Invalid character' if invalid? end
operator?()
click to toggle source
# File lib/arcamark/lexer.rb, line 66 def operator? !eof? && CharType.operator?(@curr) end
start_token()
click to toggle source
# File lib/arcamark/lexer.rb, line 82 def start_token if eof? @state = :eof elsif newline? @state = :newline elsif whitespace? @state = :whitespace elsif operator? @state = :operator elsif funcall_start? @state = :funcall elsif text? @state = :text else raise 'Invalid state' end @value = '' @value_index = @curr_index @value_line = @curr_line @value_column = @curr_column end
text?()
click to toggle source
# File lib/arcamark/lexer.rb, line 78 def text? !eof? && CharType.text?(@curr) end
whitespace?()
click to toggle source
# File lib/arcamark/lexer.rb, line 62 def whitespace? !eof? && CharType.whitespace?(@curr) end