class Sinew::Base

Sinew base class, for in standalone scripts or via the sinew binary.

Constants

GREEN
RED
RESET

stdout

Attributes

csv[R]
mutex[R]
options[R]

Public Class Methods

new(opts = {}) click to toggle source
# File lib/sinew/base.rb, line 12
def initialize(opts = {})
  @mutex = Mutex.new

  #
  # defaults for Sloptions
  #

  # default :rate_limit, typically 1
  default_rate_limit = ENV['SINEW_TEST'] ? 0 : 1

  #
  # note: uses HTTPDisk::Sloptions
  #

  @options = HTTPDisk::Sloptions.parse(opts) do
    # cli
    _1.integer :limit
    _1.integer :timeout, default: 30
    _1.boolean :silent
    _1.on :proxy, type: [:string, Array]
    _1.boolean :verbose

    # httpdisk
    _1.string :dir, default: File.join(ENV['HOME'], '.sinew')
    _1.integer :expires
    _1.boolean :force
    _1.boolean :force_errors
    _1.array :ignore_params

    # more runtime options
    _1.hash :headers
    _1.boolean :insecure
    _1.string :output, required: true
    _1.hash :params
    _1.float :rate_limit, default: default_rate_limit
    _1.integer :retries, default: 2
    _1.on :url_prefix, type: [:string, URI]
    _1.boolean :utf8, default: true
  end

  @csv = CSV.new(opts[:output])
end

Public Instance Methods

banner(msg, color: GREEN) click to toggle source

Print a nice green banner.

cached?(method, url, params = nil, body = nil) click to toggle source

Returns true if request is cached. Defaults to form body type.

# File lib/sinew/base.rb, line 94
def cached?(method, url, params = nil, body = nil)
  status = status(method, url, params, body)
  status[:status] != 'miss'
end
csv_emit(row) click to toggle source

Output a csv row. Row should be any object that can turn into a hash - a hash, OpenStruct, etc.

# File lib/sinew/base.rb, line 135
def csv_emit(row)
  row = row.to_h
  mutex.synchronize do
    # header if necessary
    csv_header(row.keys) if !csv.started?

    # emit
    print = csv.emit(row)
    puts print.ai if options[:verbose]

    # this is caught by Sinew::Main
    if csv.count == options[:limit]
      raise LimitError
    end
  end
end
csv_header(*columns) click to toggle source

Output a csv header. This usually happens automatically, but you can call this method directly to ensure a consistent set of columns.

# File lib/sinew/base.rb, line 129
def csv_header(*columns)
  csv.start(columns.flatten)
end
faraday() click to toggle source

Faraday connection for this recipe

# File lib/sinew/base.rb, line 83
def faraday
  mutex.synchronize do
    @faraday ||= create_faraday
  end
end
fatal(msg) click to toggle source

Print a scary red banner and exit.

# File lib/sinew/base.rb, line 169
def fatal(msg)
  banner(msg, color: RED)
  exit 1
end
get(url, params = nil, headers = nil) click to toggle source

http get, returns a Response

# File lib/sinew/base.rb, line 60
def get(url, params = nil, headers = nil)
  faraday_response = faraday.get(url, params, headers) do
    _1.options[:proxy] = random_proxy
  end
  Response.new(faraday_response)
end
post(url, body = nil, headers = nil) click to toggle source

http post, returns a Response. Defaults to form body type.

# File lib/sinew/base.rb, line 68
def post(url, body = nil, headers = nil)
  faraday_response = faraday.post(url, body, headers) do
    _1.options[:proxy] = random_proxy
  end
  Response.new(faraday_response)
end
post_json(url, body = nil, headers = nil) click to toggle source

http post json, returns a Response

# File lib/sinew/base.rb, line 76
def post_json(url, body = nil, headers = nil)
  body = body.to_json
  headers = (headers || {}).merge('Content-Type' => 'application/json')
  post(url, body, headers)
end
status(method, url, params = nil, body = nil) click to toggle source

Check httpdisk status for this request. Defaults to form body type.

# File lib/sinew/base.rb, line 107
def status(method, url, params = nil, body = nil)
  # if hash, default to url encoded form
  # see lib/faraday/request/url_encoded.rb
  if body.is_a?(Hash)
    body = Faraday::Utils::ParamsHash[body].to_query
  end

  env = Faraday::Env.new.tap do
    _1.method = method.to_s.downcase.to_sym
    _1.request_headers = {}
    _1.request_body = body
    _1.url = faraday.build_url(url, params)
  end
  httpdisk.status(env)
end
uncache(method, url, params = nil, body = nil) click to toggle source

Remove cache file, if any. Defaults to form body type.

# File lib/sinew/base.rb, line 100
def uncache(method, url, params = nil, body = nil)
  status = status(method, url, params, body)
  path = status[:path]
  File.unlink(path) if File.exist?(path)
end

Protected Instance Methods

create_faraday() click to toggle source

Create the Faraday connection for making requests.

# File lib/sinew/base.rb, line 186
def create_faraday
  faraday_options = options.slice(:headers, :params)
  if options[:insecure]
    faraday_options[:ssl] = { verify: false }
  end
  Faraday.new(nil, faraday_options) do
    # options
    if options[:url_prefix]
      _1.url_prefix = options[:url_prefix]
    end
    _1.options.timeout = options[:timeout]

    #
    # middleware that runs on both disk/network requests
    #

    # cookie middleware
    _1.use :cookie_jar

    # auto-encode form bodies
    _1.request :url_encoded

    # Before httpdisk so each redirect segment is cached
    # Keep track of redirect status for logger
    _1.response :follow_redirects, callback: ->(_old_env, new_env) { new_env[:redirect] = true }

    #
    # httpdisk
    #

    httpdisk_options = options.slice(:dir, :expires, :force, :force_errors, :ignore_params, :utf8)
    _1.use :httpdisk, httpdisk_options

    #
    # middleware below only used it httpdisk uses the network
    #

    # rate limit
    rate_limit = options[:rate_limit]
    _1.request :rate_limiter, interval: rate_limit

    # After httpdisk so that only non-cached requests are logged.
    # Before retry so that we don't log each retry attempt.
    _1.response :logger, nil, formatter: Middleware::LogFormatter if !options[:silent]

    retry_options = {
      max_interval: rate_limit, # very important, negates Retry-After: 86400
      max: options[:retries],
      methods: %w[delete get head options patch post put trace],
      retry_statuses: (500..600).to_a,
      retry_if: ->(_env, _err) { true },
    }
    _1.request :retry, retry_options
  end
end
httpdisk() click to toggle source

find connection's httpdisk instance

# File lib/sinew/base.rb, line 243
def httpdisk
  @httpdisk ||= begin
    app = faraday.app
    app = app.app until app.is_a?(HTTPDisk::Client)
    app
  end
end
random_proxy() click to toggle source

Return a random proxy.

# File lib/sinew/base.rb, line 177
def random_proxy
  return if !options[:proxy]

  proxies = options[:proxy]
  proxies = proxies.split(',') if !proxies.is_a?(Array)
  proxies.sample
end