class Rbexy::Lexer
Constants
- Patterns
Attributes
curr_default_text[RW]
curr_expr[RW]
curr_expr_bracket_levels[RW]
curr_expr_quote_levels[R]
curr_quoted_text[RW]
scanner[R]
stack[R]
tokens[R]
Public Class Methods
new(code)
click to toggle source
# File lib/rbexy/lexer.rb, line 40 def initialize(code) @stack = [:default] @curr_expr_bracket_levels = 0 @curr_expr_quote_levels = { single: 0, double: 0 } @curr_expr = "" @curr_default_text = "" @curr_quoted_text = "" @tokens = [] @scanner = StringScanner.new(code) end
Public Instance Methods
expression_content?()
click to toggle source
# File lib/rbexy/lexer.rb, line 279 def expression_content? # Patterns.expression_content ends at `<` characters, because we need to # separately scan for allowed open_tag_defs within expressions. We should # support any found open_tag_ends as expression content, as that means the # open_tag_def was not considered allowed (or stack would be inside # :tag_def instead of :expression) so we should thus also consider the # open_tag_end to just be a part of the expression (maybe its in a string, # etc). scanner.scan(Patterns.expression_content) || scanner.scan(Patterns.open_tag_end) end
expression_inner_bracket()
click to toggle source
# File lib/rbexy/lexer.rb, line 259 def expression_inner_bracket self.curr_expr += scanner.matched stack.push(:expression_inner_bracket) end
expression_inner_double_quote()
click to toggle source
# File lib/rbexy/lexer.rb, line 264 def expression_inner_double_quote self.curr_expr += scanner.matched stack.push(:expression_inner_double_quote) end
expression_inner_single_quote()
click to toggle source
# File lib/rbexy/lexer.rb, line 269 def expression_inner_single_quote self.curr_expr += scanner.matched stack.push(:expression_inner_single_quote) end
expression_quoted_string_content()
click to toggle source
# File lib/rbexy/lexer.rb, line 274 def expression_quoted_string_content self.curr_expr += scanner.getch stack.pop unless curr_expr.end_with?('\\') end
open_expression()
click to toggle source
# File lib/rbexy/lexer.rb, line 254 def open_expression tokens << [:OPEN_EXPRESSION] stack.push(:expression) end
open_tag_def()
click to toggle source
# File lib/rbexy/lexer.rb, line 249 def open_tag_def tokens << [:OPEN_TAG_DEF] stack.push(:tag, :tag_def) end
potential_expression_inner_tag()
click to toggle source
# File lib/rbexy/lexer.rb, line 239 def potential_expression_inner_tag if self.curr_expr =~ Patterns.expression_internal_tag_prefixes tokens << [:EXPRESSION_BODY, curr_expr] self.curr_expr = "" open_tag_def else self.curr_expr += scanner.matched end end
tokenize()
click to toggle source
# File lib/rbexy/lexer.rb, line 51 def tokenize until scanner.eos? case stack.last when :default if scanner.scan(Patterns.declaration) tokens << [:DECLARATION, scanner.matched] elsif scanner.scan(Patterns.open_tag_def) open_tag_def elsif scanner.scan(Patterns.open_expression) open_expression elsif scanner.scan(Patterns.comment) tokens << [:SILENT_NEWLINE] elsif scanner.check(Patterns.text_content) stack.push(:default_text) else raise SyntaxError, self end when :tag if scanner.scan(Patterns.open_tag_def) open_tag_def elsif scanner.scan(Patterns.open_tag_end) tokens << [:OPEN_TAG_END] stack.push(:tag_end) elsif scanner.scan(Patterns.open_expression) open_expression elsif scanner.scan(Patterns.comment) tokens << [:SILENT_NEWLINE] elsif scanner.check(Patterns.text_content) stack.push(:default_text) else raise SyntaxError, self end when :default_text if scanner.scan(Patterns.text_content) self.curr_default_text += scanner.matched if scanner.matched.end_with?('\\') && scanner.peek(1) == "{" self.curr_default_text += scanner.getch elsif scanner.matched.end_with?('\\') && scanner.peek(1) == "#" self.curr_default_text += scanner.getch else if scanner.peek(1) == "#" # If the next token is a comment, trim trailing whitespace from # the text value so we don't add to the indentation of the next # value that is output after the comment self.curr_default_text = curr_default_text.gsub(/^\p{Blank}*\z/, "") end tokens << [:TEXT, curr_default_text] self.curr_default_text = "" stack.pop end else raise SyntaxError, self end when :expression if scanner.scan(Patterns.close_expression) tokens << [:EXPRESSION_BODY, curr_expr] tokens << [:CLOSE_EXPRESSION] self.curr_expr = "" stack.pop elsif scanner.scan(Patterns.open_expression) expression_inner_bracket elsif scanner.scan(Patterns.double_quote) expression_inner_double_quote elsif scanner.scan(Patterns.single_quote) expression_inner_single_quote elsif scanner.scan(Patterns.open_tag_def) potential_expression_inner_tag elsif expression_content? self.curr_expr += scanner.matched else raise SyntaxError, self end when :expression_inner_bracket if scanner.scan(Patterns.close_expression) self.curr_expr += scanner.matched stack.pop elsif scanner.scan(Patterns.open_expression) expression_inner_bracket elsif scanner.scan(Patterns.double_quote) expression_inner_double_quote elsif scanner.scan(Patterns.single_quote) expression_inner_single_quote elsif scanner.scan(Patterns.open_tag_def) potential_expression_inner_tag elsif expression_content? self.curr_expr += scanner.matched else raise SyntaxError, self end when :expression_inner_double_quote if scanner.check(Patterns.double_quote) expression_quoted_string_content elsif scanner.scan(Patterns.double_quoted_text_content) self.curr_expr += scanner.matched else raise SyntaxError, self end when :expression_inner_single_quote if scanner.check(Patterns.single_quote) expression_quoted_string_content elsif scanner.scan(Patterns.single_quoted_text_content) self.curr_expr += scanner.matched else raise SyntaxError, self end when :tag_def if scanner.scan(Patterns.close_self_closing_tag) tokens << [:CLOSE_TAG_DEF] tokens << [:OPEN_TAG_END] tokens << [:CLOSE_TAG_END] stack.pop(2) elsif scanner.scan(Patterns.close_tag) tokens << [:CLOSE_TAG_DEF] stack.pop elsif scanner.scan(Patterns.tag_name) tokens << [:TAG_NAME, scanner.matched] elsif scanner.scan(Patterns.whitespace) scanner.matched.count("\n").times { tokens << [:SILENT_NEWLINE] } tokens << [:OPEN_ATTRS] stack.push(:tag_attrs) else raise SyntaxError, self end when :tag_end if scanner.scan(Patterns.close_tag) tokens << [:CLOSE_TAG_END] stack.pop(2) elsif scanner.scan(Patterns.tag_name) tokens << [:TAG_NAME, scanner.matched] else raise SyntaxError, self end when :tag_attrs if scanner.scan(Patterns.whitespace) scanner.matched.count("\n").times { tokens << [:SILENT_NEWLINE] } elsif scanner.check(Patterns.close_tag) tokens << [:CLOSE_ATTRS] stack.pop elsif scanner.scan(Patterns.attr_assignment) tokens << [:OPEN_ATTR_VALUE] stack.push(:tag_attr_value) elsif scanner.scan(Patterns.attr) tokens << [:ATTR_NAME, scanner.matched.strip] elsif scanner.scan(Patterns.open_attr_splat) tokens << [:OPEN_ATTR_SPLAT] tokens << [:OPEN_EXPRESSION] stack.push(:tag_attr_splat, :expression) else raise SyntaxError, self end when :tag_attr_value if scanner.scan(Patterns.double_quote) stack.push(:quoted_text) elsif scanner.scan(Patterns.open_expression) open_expression elsif scanner.scan(Patterns.whitespace) || scanner.check(Patterns.close_tag) tokens << [:CLOSE_ATTR_VALUE] scanner.matched.count("\n").times { tokens << [:SILENT_NEWLINE] } stack.pop else raise SyntaxError, self end when :tag_attr_splat # Splat is consumed by :expression. It pops control back to here once # it's done, and we just record the completion and pop back to :tag_attrs tokens << [:CLOSE_ATTR_SPLAT] stack.pop when :quoted_text if scanner.scan(Patterns.double_quoted_text_content) self.curr_quoted_text += scanner.matched if scanner.matched.end_with?('\\') && scanner.peek(1) == "\"" self.curr_quoted_text += scanner.getch end elsif scanner.scan(Patterns.double_quote) tokens << [:TEXT, curr_quoted_text] self.curr_quoted_text = "" stack.pop else raise SyntaxError, self end else raise SyntaxError, self end end tokens end