class Tokipona::Tokenizer

Splits text into tokens (words and punctuations).

@example

Tokipona::Tokenizer.tokenize("mi pona anu seme?")
# => ["mi", "pona", "anu", "seme", "?"]

Public Class Methods

new(text) click to toggle source
# File lib/tokipona/tokenizer.rb, line 15
def initialize(text)
  @text = text
end
tokenize(text) click to toggle source

@param text [String]

@return [Array<String>]

# File lib/tokipona/tokenizer.rb, line 11
def self.tokenize(text)
  new(text).tokenize
end

Public Instance Methods

tokenize() click to toggle source
# File lib/tokipona/tokenizer.rb, line 19
def tokenize
  @text.scan(/\w+|[^\s]/)
end