class GameLanguage
macro
BLANK [\ \t]+ #only works on macro NUMBER \d+|-\d+ FLOAT \d+\.\d+|-\d+\.\d+ WORD [\ \t]+[a-z]+|[\ \t]+[A-Z][a-z]+|[\ \t]+[a-z]+\d+|[\ \t]+[A-Z][a-z]+\d+ FUNCTION \w+
rule
{WORD} { [:WORD, text.gsub!(/[\ \t]+/,'')] } {BLANK} #do nothing {NUMBER} {[:NUMBER, text.to_i]} {FLOAT} {[:FLOAT, test.to_f]} {FUNCTION} {[:FUNCTION, text]}
inner
def tokenize(code) scan_setup(code) tokens = [] while token = next_token tokens << token end tokens end
end