class DhEasy::Qa::ValidateScraper

Attributes

collections[R]
options[R]
outputs[R]
rules[R]
scraper_name[R]

Public Class Methods

new(scraper_name, collections, rules, outputs, thresholds) click to toggle source
# File lib/dh_easy/qa/validate_internal.rb, line 40
def initialize(scraper_name, collections, rules, outputs, thresholds)
  @scraper_name = scraper_name
  @collections = collections
  @rules = rules
  @outputs = outputs
  @options = {}
  options['thresholds'] = thresholds[scraper_name] if thresholds && thresholds[scraper_name]
end

Public Instance Methods

run() click to toggle source
# File lib/dh_easy/qa/validate_internal.rb, line 49
def run
  begin
    output_scraper
    if status_ok?
      validate_collections if collections && collections.any?
    else
      output_response
      return nil
    end
  rescue StandardError => e
    puts "An error has occurred for the scraper named '#{scraper_name}': #{e}"
    return nil
  end
end

Private Instance Methods

collection_counts() click to toggle source
# File lib/dh_easy/qa/validate_internal.rb, line 93
def collection_counts
  @collection_counts ||= collection_response.parsed_response
end
collection_response() click to toggle source
# File lib/dh_easy/qa/validate_internal.rb, line 97
def collection_response
  @collection_response || Datahen::Client::ScraperJobOutput.new.collections(scraper_name)
end
output_response() click to toggle source
# File lib/dh_easy/qa/validate_internal.rb, line 85
def output_response
  if collection_response.parsed_response.nil?
    puts "collection response is null"
  else
    puts collection_response.parsed_response['message']
  end
end
output_scraper() click to toggle source
# File lib/dh_easy/qa/validate_internal.rb, line 66
def output_scraper
  puts "validating scraper: #{scraper_name}"
end
status_ok?() click to toggle source
# File lib/dh_easy/qa/validate_internal.rb, line 70
def status_ok?
  !collection_response.parsed_response.nil? && collection_response.code == 200
end
validate_collections() click to toggle source
# File lib/dh_easy/qa/validate_internal.rb, line 74
def validate_collections
  collections.each do |collection_name|
    collection = collection_counts.find{|collection_hash| collection_hash['collection'] == collection_name }
    if collection
      ValidateCollection.new(scraper_name, collection_name, collection['outputs'], rules, outputs, options).run
    else
      puts "collection #{collection_name} is missing"
    end
  end
end