class Omnium::Lexer::Core
The lexer returns tokens for a given text.
Constants
- ALPHA
- COMMENT
- DECIMAL_POINT
- IDENTIFIER
- INTEGER
- NEWLINE
- WHITESPACE
Public Class Methods
new(text)
click to toggle source
# File lib/omnium/lexer/core.rb, line 23 def initialize(text) @text = text @pointer = 0 end
Public Instance Methods
next_token()
click to toggle source
# File lib/omnium/lexer/core.rb, line 28 def next_token ignore return new_eof_token if eos? (@pointer...@text.length).each do return tokenise end end
Private Instance Methods
advance(n = 1)
click to toggle source
# File lib/omnium/lexer/core.rb, line 52 def advance(n = 1) @pointer += n end
assignment?()
click to toggle source
# File lib/omnium/lexer/core.rb, line 116 def assignment? colon = assignment_token(:value)[0] equals = assignment_token(:value)[1] character == colon && peek == equals end
character()
click to toggle source
# File lib/omnium/lexer/core.rb, line 56 def character @character = @text[@pointer] end
comment?()
click to toggle source
# File lib/omnium/lexer/core.rb, line 70 def comment? character == COMMENT end
decimal?()
click to toggle source
# File lib/omnium/lexer/core.rb, line 78 def decimal? character == DECIMAL_POINT end
eos?()
click to toggle source
# File lib/omnium/lexer/core.rb, line 48 def eos? @pointer > @text.length - 1 end
ignore()
click to toggle source
# File lib/omnium/lexer/core.rb, line 39 def ignore advance while whitespace? || newline? if comment? advance until newline? # comment text ignore if whitespace? || newline? # skip any other junk end end
newline?()
click to toggle source
# File lib/omnium/lexer/core.rb, line 74 def newline? character == NEWLINE end
number()
click to toggle source
# File lib/omnium/lexer/core.rb, line 82 def number result = '' while character =~ INTEGER result += character advance end if decimal? result += character advance while character =~ INTEGER result += character advance end return result.to_f end result.to_i end
number_token(num)
click to toggle source
# File lib/omnium/lexer/core.rb, line 131 def number_token(num) return new_integer_token(num) if num.is_a? Integer new_real_token(num) end
peek()
click to toggle source
# File lib/omnium/lexer/core.rb, line 60 def peek return nil if eos? @text[@pointer + 1] end
reserved_keyword()
click to toggle source
# File lib/omnium/lexer/core.rb, line 105 def reserved_keyword result = '' while character =~ IDENTIFIER result += character advance end result end
tokenise()
click to toggle source
# File lib/omnium/lexer/core.rb, line 137 def tokenise if character =~ ALPHA word_token(reserved_keyword) elsif character =~ INTEGER number_token(number) elsif plus? advance new_plus_token elsif minus? advance new_minus_token elsif multiply? advance new_multiply_token elsif divide? advance new_divide_token elsif left_parenthesis? advance new_left_parenthesis_token elsif right_parenthesis? advance new_right_parenthesis_token elsif assignment? advance(2) new_assignment_token elsif semicolon? advance new_semicolon_token elsif colon? advance new_colon_token elsif comma? advance new_comma_token elsif dot? advance new_dot_token else raise(LexerError, "Error tokenising '#{character}' at position #{@pointer}") end end
whitespace?()
click to toggle source
# File lib/omnium/lexer/core.rb, line 66 def whitespace? character == WHITESPACE end
word_token(word)
click to toggle source
# File lib/omnium/lexer/core.rb, line 123 def word_token(word) if RESERVED_KEYWORDS.include?(word.intern) send("new_#{word}_token") else new_identifier_token(word) end end