class Botz::Shell

botz shell interface

Attributes

definition_file[R]

Public Class Methods

new(definition_file) click to toggle source
# File lib/botz/shell.rb, line 10
def initialize(definition_file)
  @definition_file = definition_file
end

Public Instance Methods

build(name) click to toggle source

rubocop:disable Metrics/MethodLength

# File lib/botz/shell.rb, line 54
  def build(name)
    File.open("#{name}.rb", 'w') do |f|
      f.write <<~RUBY
        # frozen_string_literal: true

        Botz.define(:#{name}) do
          spider(:example, 'http://example.com') do |html, yielder|
            #  yielder.call(url or resource)
          end

          scraper(:example) do
          end
        end
      RUBY
    end

    File.open("#{name}.sh", 'w') do |f|
      f.write <<~SHELL
        #!/bin/bash
        eval "$(botz $(dirname "${0}")/#{name}.rb shell)"
        spider example
      SHELL
    end
  end
function() click to toggle source
# File lib/botz/shell.rb, line 42
  def function
    print <<~SHELL
      function spider() {
        botz spider #{definition_file.path} $1
      }
      function scraper() {
        botz scraper #{definition_file.path} $1
      }
    SHELL
  end
scraper(name) click to toggle source

rubocop:disable Lint/AssignmentInCondition, Style/RescueStandardError

# File lib/botz/shell.rb, line 15
def scraper(name)
  command = scrapers[name.to_sym]
  fail "undefined commmand[#{name}]" if command.nil?

  while line = STDIN.gets
    url = line.strip
    begin
      command.call(url, &definition_file.output)
    rescue => e
      STDERR.puts "ERROR #{url}: #{e}\n#{e.backtrace}"
    end
  end
end
spider(name) click to toggle source

rubocop:enable Lint/AssignmentInCondition, Style/RescueStandardError

# File lib/botz/shell.rb, line 30
def spider(name)
  command = spiders[name.to_sym]
  if File.pipe?(STDIN)
    STDIN.each_line do |line|
      start_url = line.strip
      command.call(start_url) { |url| puts url }
    end
  else
    command.call { |url| puts url }
  end
end