class Chef::Knife::Mirror

Extending Chef Knife with our Mirror additions

Public Instance Methods

run() click to toggle source
# File lib/chef/knife/mirror.rb, line 66
def run
  $stdout.sync = true # Avoid possible buffering of some progress dots
  if @name_args[0] == 'all'
    # Mirror all cookbooks from SUPERMARKET_SITE to TARGET_SUPERMARKET_SITE
    ui.info('Mirroring all versions for all cookbooks')
    ui.info("Delaying by #{config[:delay]} seconds per version.") if config[:delay]
    print "Fetching cookbook index from #{config[:supermarket_site]} (source)... "
    community_universe = unauthenticated_get_rest("#{config[:supermarket_site]}/universe")
    ui.info('Done!')
    print "Fetching cookbook index from #{config[:target_site]} (target)... "
    private_universe = unauthenticated_get_rest("#{config[:target_site]}/universe")
    ui.info('Done!')
    removed, added = universe_diff(private_universe, community_universe)
    ui.info("We are still missing #{added.size} cookbooks (out of #{community_universe.size}) on our target Supermarket.")
    ui.info("Though we're not doing anything with these just yet, you should know we have #{removed.size} cookbooks which are no longer present on the Supermarket (source).") if removed.size > 0
    added.sort.each do |cookbook, versions|
      ui.info("Mirroring #{versions.size} version(s) for #{cookbook} cookbook:")
      displayed_before = false
      versions.sort_by { |version, _versionmeta| version.split('.').map(&:to_i) }.each do |version, _versionmeta|
        mirror_cookbook(cookbook, version, displayed_before)
        displayed_before = true # Some things only need displaying once per cookbook and not each version (deprecation)
        sleep(config[:delay].to_i) if config[:delay]
      end
    end
  elsif @name_args[1] == 'all'
    # Mirror all versions for a specific cookbook
    ui.info("Mirroring all versions for #{@name_args[0]}.")
    ui.info("Delaying by #{config[:delay]} seconds per version.") if config[:delay]
    print "Fetching cookbook meta from #{config[:supermarket_site]} (source)... "
    cookbookmeta = get_cookbook_meta
    ui.info('Done!')
    print "Fetching cookbook meta from #{config[:target_site]} (target)... "
    target_cookbookmeta = get_cookbook_meta(@name_args[0], "#{config[:target_site]}/api/v1/cookbooks")
    ui.info('Done!')
    ui.info("Processing remaining #{cookbookmeta['metrics']['downloads']['versions'].size - target_cookbookmeta['metrics']['downloads']['versions'].size} cookbook versions:")
    displayed_before = false
    (cookbookmeta['metrics']['downloads']['versions'].keys - target_cookbookmeta['metrics']['downloads']['versions'].keys).sort_by { |version| version.split('.').map(&:to_i) }.each do |version|
      mirror_cookbook(@name_args[0], version, displayed_before)
      displayed_before = true # Some things only need displaying once per cookbook and not each version (deprecation)
      sleep(config[:delay].to_i) if config[:delay]
    end
  elsif @name_args.length == 2
    # Mirror just one single cookbook, specific version
    ui.info("Mirroring #{@name_args[0]} (#{@name_args[1]}).")
    mirror_cookbook(@name_args[0], @name_args[1])
  else
    # Mirror just one single cookbook, latest version
    ui.info("Mirroring #{@name_args[0]} (latest version).")
    mirror_cookbook
    if config[:deps]
      ui.info('Mirroring dependencies as well...')
      print "Fetching cookbook index from #{config[:supermarket_site]} (source)... "
      universe = unauthenticated_get_rest("#{config[:supermarket_site]}/universe")
      ui.info('Done!')
      get_cookbook_version_meta['dependencies'].each do |cookbook, version_constraint|
        universe[cookbook].sort_by { |version, _versionmeta| version.split('.').map(&:to_i) }.reverse!.each do |version, _versionmeta|
          next unless Chef::VersionConstraint.new(version_constraint).include?(version)
          ui.info("Most recent version matching constraint (#{version_constraint}) for cookbook #{cookbook}: #{version}")
          mirror_cookbook(cookbook, version)
          sleep(config[:delay].to_i) if config[:delay]
          break
        end
      end
    end
  end
end

Private Instance Methods

cookbook_deprecated?(cookbookmeta) click to toggle source
# File lib/chef/knife/mirror.rb, line 195
def cookbook_deprecated?(cookbookmeta)
  cookbookmeta['deprecated'] == true
end
get_cookbook_meta(cookbook = @name_args[0], api_url = " click to toggle source
# File lib/chef/knife/mirror.rb, line 178
def get_cookbook_meta(cookbook = @name_args[0], api_url = "#{config[:supermarket_site]}/api/v1/cookbooks")
  unauthenticated_get_rest("#{api_url}/#{cookbook}")
rescue => e
  if e.message =~ /404/
    # Return proper (empty) metadata structure
    md = Chef::Cookbook::Metadata.new
    return md.to_hash.merge!('metrics' => { 'downloads' => { 'versions' => {} } })
  else
    ui.error("Error during #{cookbook} metadata request (#{e.message}). Increase log verbosity (-VV) for more information.")
    exit(1)
  end
end
get_cookbook_version_meta(cookbook = @name_args[0], version = 'latest_version', api_url = " click to toggle source
# File lib/chef/knife/mirror.rb, line 191
def get_cookbook_version_meta(cookbook = @name_args[0], version = 'latest_version', api_url = "#{config[:supermarket_site]}/api/v1/cookbooks")
  version == 'latest_version' ? unauthenticated_get_rest(get_cookbook_meta(cookbook, api_url)[version]) : unauthenticated_get_rest("#{api_url}/#{cookbook}/versions/#{version.gsub('.', '_')}")
end
mirror_cookbook(cookbook = @name_args[0], version = 'latest_version', displayed_before = false, user_id = Chef::Config[:node_name], user_secret_filename = Chef::Config[:client_key]) click to toggle source
# File lib/chef/knife/mirror.rb, line 135
def mirror_cookbook(cookbook = @name_args[0], version = 'latest_version', displayed_before = false, user_id = Chef::Config[:node_name], user_secret_filename = Chef::Config[:client_key])
  cookbookmeta = get_cookbook_meta(cookbook, "#{config[:supermarket_site]}/api/v1/cookbooks")
  ui.warn("This cookbook has been deprecated. It has been replaced by #{File.basename(cookbookmeta['replacement'])}.") if cookbook_deprecated?(cookbookmeta) && !displayed_before
  versionmeta = version == 'latest_version' ? unauthenticated_get_rest(cookbookmeta['latest_version']) : unauthenticated_get_rest("#{config[:supermarket_site]}/api/v1/cookbooks/#{cookbook}/versions/#{version.gsub('.', '_')}")
  print "Processing version #{versionmeta['version']} "
  temp_cookbookfile = unauthenticated_get_rest(versionmeta['file'], true)
  print '.'
  # Need to revisit this, currently Supermarket only support setting the catagory this way :(
  # meta_hash = %w(category external_url source_url issues_url average_rating created_at up_for_adoption foodcritic_failure).map { |param| [param.to_sym, cookbookmeta[param]] }.to_h
  # meta_hash.merge!('deprecated' => true, 'replacement' => "#{config[:target_site]}/api/v1/cookbooks/#{File.basename(cookbookmeta['replacement'])}") if cookbook_deprecated?(cookbookmeta)
  # So for now, we just fix the category :(
  meta_hash = { 'category' => '' }
  begin
    http_resp = Chef::CookbookSiteStreamingUploader.post("#{config[:target_site]}/api/v1/cookbooks", user_id, user_secret_filename, tarball: File.open(temp_cookbookfile.path), cookbook: meta_hash.to_json)
  rescue => e
    ui.error("Error uploading cookbook #{cookbook} (#{versionmeta['version']}) to the Supermarket at #{config[:target_site]}: #{e.message}. Increase log verbosity (-VV) for more information.")
    Chef::Log.debug("\n#{e.backtrace.join("\n")}")
    config[:keep] ? FileUtils.mv(temp_cookbookfile.path, File.join(config[:download_directory], "#{cookbook}-#{versionmeta['version']}.tar.gz")) : FileUtils.rm_rf(temp_cookbookfile.path)
    exit(1) # Hard exit since this usually hints at trouble reaching the supermarket, no sense in allowing this in some loop...
  end
  print '.'
  if http_resp.code.to_i != 201
    res = Chef::JSONCompat.from_json(http_resp.body) unless http_resp.code.to_i == 500
    ui.info('. Failed :(')
    if http_resp.code.to_i != 500 && res['error_messages']
      ui.error "#{res['error_messages'][0]}"
    else
      ui.error 'Unknown error while uploading cookbook'
      ui.error "Server response: #{http_resp.body}"
    end
    config[:keep] ? FileUtils.mv(temp_cookbookfile.path, File.join(config[:download_directory], "#{cookbook}-#{versionmeta['version']}.tar.gz")) : FileUtils.rm_rf(temp_cookbookfile.path)
    ui.info("Saving failed cookbook (#{File.join(config[:download_directory], "#{cookbook}-#{versionmeta['version']}.tar.gz")})") if config[:keep]
    Chef::Log.debug("Removing #{temp_cookbookfile.path}") unless config[:keep]
    return
  end
  ui.info('. Done!')
end
skip?(obj) click to toggle source
# File lib/chef/knife/mirror.rb, line 220
def skip?(obj)
  obj.is_a?(Hash) ? (obj.empty? || %w(location_path download_url).any? { |key| obj.key?(key) }) : obj.nil?
end
unauthenticated_get_rest(url, raw = false) click to toggle source
# File lib/chef/knife/mirror.rb, line 173
def unauthenticated_get_rest(url, raw = false)
  noauth_rest.sign_on_redirect = false
  noauth_rest.get_rest(url, raw)
end
universe_diff(source, target) click to toggle source
# File lib/chef/knife/mirror.rb, line 199
def universe_diff(source, target)
  return [nil, target.dup] if source.nil?
  return [source.dup, nil] if target.nil?
  if source.is_a?(Hash) && target.is_a?(Hash)
    added = {}
    removed = {}
    source_keys = source.keys
    target_keys = target.keys
    (source_keys - target_keys).each { |key| removed[key] = source[key].dup }
    (target_keys - source_keys).each { |key| added[key] = target[key].dup }
    (source_keys & target_keys).each do |key|
      nested_removed, nested_added = universe_diff(source[key], target[key])
      removed[key] = nested_removed unless skip?(nested_removed)
      added[key] = nested_added unless skip?(nested_added)
    end
    [removed, added]
  elsif source != target
    [source, target]
  end
end