class VueCompiler::Tokenizer

Constants

ATTRIBUTE_NAME_RE
ATTRIBUTE_VALUE_RE
QUOTE_RE
TAG_NAME_RE
TAG_RE
VOID_TAG
WHITESPACE_RE

Public Class Methods

new(code) click to toggle source

@param code {String}

# File lib/vue_component_compiler/tokenizer.rb, line 13
def initialize(code)
  @code = code.delete_prefix("\uFEFF")
  @tokens = []
end

Public Instance Methods

eat(str) click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 39
def eat(str)
  @code.delete_prefix!(str)
end
eat_whitespaces() click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 43
def eat_whitespaces
  @code.sub!(WHITESPACE_RE, '')
end
expect(str) click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 47
def expect(str)
  if @code.start_with?(str)
    self.eat(str)
  elsif
    raise Exception.new("Found character `#{@code[0]}`, expecting `#{str}`")
  end
end
scan_attribute() click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 145
def scan_attribute
  name = @code.slice(ATTRIBUTE_NAME_RE)
  value = nil
  @tokens << Token::AttributeName.new(name)
  self.eat(name)

  self.eat_whitespaces
  return { name: name, value: value } unless @code.start_with?('=')

  self.expect('=')
  self.eat_whitespaces
  if (@code =~ ATTRIBUTE_VALUE_RE) == 0
    self.scan_attribute_value
    value = self.eat_whitespaces
  end

  { name: name, value: value }
end
scan_attribute_value() click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 164
def scan_attribute_value
  quoted = @code.start_with?("'") || @code.start_with?('"')
  matched = @code.match(ATTRIBUTE_VALUE_RE)
  value = matched[1] || matched[2] || matched[3]
  @tokens << Token::AttributeValue.new(value)

  @code = @code[((quoted ? 2 : 0) + value.size)..-1]

  value
end
scan_comment() click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 208
def scan_comment
  @tokens << Token::CommentStart.new
  end_pos = @code.index('-->')
  if end_pos
    @tokens << Token::Comment.new(@code[4...end_pos])
    @tokens << Token::CommentEnd.new
    @code = @code[end_pos + 3..-1]
  end
end
scan_custom_block() click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 299
def scan_custom_block
  self.eat('<')
  type = @code.slice(TAG_NAME_RE)
  content = ''
  self.eat(type)

  self.eat_whitespaces
  # TODO: support attributes on custom block.
  self.eat('>')

  until @code.start_with?("</#{type}")
    char = @code[0]
    content << char
    self.eat(char)
  end

  self.eat("</#{type}")
  self.eat_whitespaces
  self.eat('>')

  @tokens << Token::CustomBlock.new(type, content)
end
scan_interpolation() click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 186
def scan_interpolation
  expr = ''
  is_string = false

  @tokens << Token::InterpolationStart.new
  self.eat('{{')

  while is_string || (!is_string && !@code.start_with?('}}'))
    char = @code[0]
    expr << char
    self.eat(char)
    if char == '"' || char == "'" || char == '`'
      is_string = !is_string
    end
  end
  expr.strip!
  @tokens << Token::Interpolation.new(expr)

  @tokens << Token::InterpolationEnd.new
  self.eat('}}')
end
scan_script_block() click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 218
def scan_script_block
  self.eat('<script')
  self.eat_whitespaces

  lang = if @code.start_with?('lang')
    self.eat('lang')
    self.eat_whitespaces
    self.expect('=')
    self.eat_whitespaces
    quoted = @code.start_with?("'") || @code.start_with?('"')
    matched = @code.match(ATTRIBUTE_VALUE_RE)
    lang = matched[1] || matched[2] || matched[3]
    @code = @code[((quoted ? 2 : 0) + lang.size)..-1]
    self.eat_whitespaces
    self.expect('>')
    lang
  elsif @code.start_with?('>')
    self.eat('>')
    nil
  end

  content = ''
  is_string = false
  while is_string || (!is_string && !@code.start_with?('</script'))
    char = @code[0]
    if char == '"' || char == "'" || char == '`'
      is_string = !is_string
    end
    content << char
    self.eat(char)
  end

  self.eat('</script')
  self.eat_whitespaces
  self.expect('>')

  @tokens << Token::ScriptBlock.new(content, lang)
end
scan_style_block() click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 257
def scan_style_block
  content = ''
  scoped = false
  lang = 'css'

  self.eat('<style')
  self.eat_whitespaces

  until @code.start_with?('>')
    if @code.start_with?('scoped')
      scoped = true
      self.eat('scoped')
    elsif @code.start_with?('lang')
      self.eat('lang')
      self.eat_whitespaces
      self.expect('=')
      self.eat_whitespaces
      quoted = @code.start_with?("'") || @code.start_with?('"')
      matched = @code.match(ATTRIBUTE_VALUE_RE)
      lang = matched[1] || matched[2] || matched[3]
      @code.sub!(QUOTE_RE, '')
      self.eat(lang)
      @code.sub!(QUOTE_RE, '')
    end
    self.eat_whitespaces
  end

  self.eat('>')

  until @code.start_with?('</style')
    char = @code[0]
    content << char
    self.eat(char)
  end

  self.eat('</style')
  self.eat_whitespaces
  self.expect('>')

  @tokens << Token::StyleBlock.new(content, lang, scoped)
end
scan_tag() click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 86
def scan_tag
  tag_name = self.scan_tag_name
  is_v_pre = false
  self.eat_whitespaces

  while (@code =~ ATTRIBUTE_NAME_RE) == 0
    attr = self.scan_attribute
    if attr[:name] == 'v-pre'
      is_v_pre = true
    end
    self.eat_whitespaces
  end

  if @code.start_with?('/>')
    @tokens << Token::SelfClosingTag.new
    self.eat('/>')
    return
  elsif @code.start_with?('>')
    if VOID_TAG.include?(tag_name)
      @tokens << Token::SelfClosingTag.new
      self.eat('>')
      return
    else
      @tokens << Token::TagEnd.new
      self.eat('>')
    end
  end

  until @code.start_with?('</')
    if @code.start_with?('<!--')
      self.scan_comment
    elsif @code.start_with?('<')
      self.scan_tag
    elsif @code.start_with?('{{')
      self.scan_interpolation
    else
      self.scan_text(is_v_pre)
    end
  end

  self.expect('</')
  @tokens << Token::ClosingTagStart.new
  self.expect(tag_name)
  @tokens << Token::ClosingTagName.new(tag_name)
  self.eat_whitespaces
  self.expect('>')
  @tokens << Token::TagEnd.new
end
scan_tag_name() click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 135
def scan_tag_name
  self.expect('<')
  @tokens << Token::OpenTagStart.new

  tag_name = @code.slice(TAG_NAME_RE)
  @tokens << Token::OpenTagName.new(tag_name)
  self.eat(tag_name)
  tag_name
end
scan_text(ignore_interpolation = false) click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 175
def scan_text(ignore_interpolation = false)
  text = ''
  until @code.start_with?('<') || (!ignore_interpolation && @code.start_with?('{{'))
    char = @code[0]
    self.eat(char)
    next if char == ' ' && text.end_with?(' ')
    text << char
  end
  @tokens << Token::Text.new(text)
end
scan_top_level_template() click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 55
def scan_top_level_template
  @tokens << Token::OpenTagStart.new
  @tokens << Token::OpenTagName.new('template')
  @tokens << Token::TagEnd.new
  self.eat('<template')
  self.eat_whitespaces
  self.expect('>')

  self.eat_whitespaces
  root_scanned = false
  until @code.start_with?('</')
    if @code.start_with?('<!--')
      self.scan_comment
    else
      unless root_scanned
        self.scan_tag
        root_scanned = true
      end
    end
    self.eat_whitespaces
  end

  self.eat('</')
  @tokens << Token::ClosingTagStart.new
  self.expect('template')
  @tokens << Token::ClosingTagName.new('template')
  self.eat_whitespaces
  self.expect('>')
  @tokens << Token::TagEnd.new
end
tokenize() click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 18
def tokenize
  while @code && !@code.empty?
    if @code.start_with?('<template')
      self.scan_top_level_template
    elsif @code =~ WHITESPACE_RE
      self.eat_whitespaces
    elsif @code.start_with?('<script')
      self.scan_script_block
    elsif @code.start_with?('<style')
      self.scan_style_block
    elsif @code.start_with?('<!--')
      self.scan_comment
    elsif @code =~ TAG_RE
      self.scan_custom_block
    end
  end
  @tokens << Token::EndOfSource.new

  @tokens
end