module Pkg::Rpm::Repo

Public Class Methods

base_url() click to toggle source
# File lib/packaging/rpm/repo.rb, line 7
def base_url
  "http://#{Pkg::Config.builds_server}/#{Pkg::Config.project}/#{Pkg::Config.ref}"
end
create_local_repos(directory = "repos") click to toggle source
# File lib/packaging/rpm/repo.rb, line 206
def create_local_repos(directory = "repos")
  stdout, _, _ = Pkg::Util::Execution.capture3("bash -c '#{repo_creation_command(directory)}'")
  stdout
end
create_remote_repos(directory = 'repos') click to toggle source
# File lib/packaging/rpm/repo.rb, line 211
def create_remote_repos(directory = 'repos')
  artifact_directory = File.join(Pkg::Config.jenkins_repo_path, Pkg::Config.project, Pkg::Config.ref)
  artifact_paths = Pkg::Repo.directories_that_contain_packages(File.join(artifact_directory, 'artifacts'), 'rpm')
  Pkg::Repo.populate_repo_directory(artifact_directory)
  command = Pkg::Rpm::Repo.repo_creation_command(File.join(artifact_directory, directory), artifact_paths)

  begin
    Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, command)
    # Now that we've created our package repositories, we can generate repo
    # configurations for use with downstream jobs, acceptance clients, etc.
    Pkg::Rpm::Repo.generate_repo_configs

    # Now that we've created the repo configs, we can ship them
    Pkg::Rpm::Repo.ship_repo_configs
  ensure
    # Always remove the lock file, even if we've failed
    Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock")
  end
end
create_repos(directory = "repos") click to toggle source
# File lib/packaging/rpm/repo.rb, line 236
def create_repos(directory = "repos")
  Pkg::Util.deprecate('Pkg::Rpm::Repo.create_repos', 'Pkg::Rpm::Repo.create_local_repos')
  create_local_repos(directory)
end
create_repos_from_artifacts(directory = "repos") click to toggle source
# File lib/packaging/rpm/repo.rb, line 231
def create_repos_from_artifacts(directory = "repos")
  Pkg::Util.deprecate('Pkg::Rpm::Repo.create_repos_from_artifacts', 'Pkg::Rpm::Repo.create_remote_repos')
  create_remote_repos(directory)
end
deploy_repos(yum_path, origin_server, destination_server, dryrun = false) click to toggle source

@deprecated this command is exactly as awful as you think it is.

-- Ryan McKern 12/2015

@param yum_path [String] path for rpm repos on local and remote filesystem @param origin_server [String] remote host to start the rsync from @param destination_server [String] remote host to send rsynced content to @param dryrun [Boolean] whether or not to use '–dry-run'

# File lib/packaging/rpm/repo.rb, line 248
def deploy_repos(yum_path, origin_server, destination_server, dryrun = false)
  rsync_command = repo_deployment_command(yum_path, yum_path, destination_server, dryrun)

  Pkg::Util::Net.remote_execute(origin_server, rsync_command)
end
generate_repo_configs(source = "repos", target = "repo_configs", signed = false) click to toggle source

Generate yum configuration files that point to the repositories created on the distribution server with packages created from the current source repo commit. There is one for each dist/version that is packaged (e.g. el5, el6, etc). Files are created in pkg/repo_configs/rpm and are named pl-$project-$sha.conf, and can be placed in /etc/yum.repos.d to enable clients to install these packages.

# File lib/packaging/rpm/repo.rb, line 134
def generate_repo_configs(source = "repos", target = "repo_configs", signed = false)
  # We have a hard requirement on wget because of all the download magicks
  # we have to do
  #
  wget = Pkg::Util::Tool.check_tool("wget")

  # This is the standard path to all build artifacts on the distribution
  # server for this commit
  #
  repo_base = "#{base_url}/#{source}/"

  # First check if the artifacts directory exists
  #

  # We have to do two checks here - first that there are directories with
  # repodata folders in them, and second that those same directories also
  # contain rpms
  #
  stdout, _, _ = Pkg::Util::Execution.capture3("#{wget} --spider -r -l 5 --no-parent #{repo_base} 2>&1")
  stdout = stdout.split.uniq.reject { |x| x =~ /\?|index/ }.select { |x| x =~ /http:.*repodata\/$/ }

  # RPMs will always exist at the same directory level as the repodata
  # folder, which means if we go up a level we should find rpms
  #
  yum_repos = []
  stdout.map { |x| x.chomp('repodata/') }.each do |url|
    output, _, _ = Pkg::Util::Execution.capture3("#{wget} --spider -r -l 1 --no-parent #{url} 2>&1")
    unless output.split.uniq.reject { |x| x =~ /\?|index/ }.select { |x| x =~ /http:.*\.rpm$/ }.empty?
      yum_repos << url
    end
  end

  if yum_repos.empty?
    warn "No rpm repos were found to generate configs from!"
    return
  end

  FileUtils.mkdir_p(File.join("pkg", target, "rpm"))

  # Parse the rpm configs file to generate repository configs. Each line in
  # the rpm_configs file corresponds with a repo directory on the
  # distribution server.
  #
  yum_repos.each do |url|
    # We ship a base 'srpm' that gets turned into a repo, but we want to
    # ignore this one because its an extra
    next if url == "#{repo_base}srpm/"

    platform_tag = Pkg::Paths.tag_from_artifact_path(url)
    platform, version, arch = Pkg::Platforms.parse_platform_tag(platform_tag)

    # Create an array of lines that will become our yum config
    #
    config = ["[pl-#{Pkg::Config.project}-#{Pkg::Config.ref}]"]
    config << ["name=PL Repo for #{Pkg::Config.project} at commit #{Pkg::Config.ref}"]
    config << ["baseurl=#{url}"]
    config << ["enabled=1"]
    if signed
      config << ["gpgcheck=1"]
      config << ["gpgkey=http://#{Pkg::Config.builds_server}/#{Pkg::Util::Gpg.key}"]
    else
      config << ["gpgcheck=0"]
    end

    # Write the new config to a file under our repo configs dir
    #
    config_file = File.join("pkg", target, "rpm", "pl-#{Pkg::Config.project}-#{Pkg::Config.ref}-#{platform}-#{version}-#{arch}.repo")
    File.open(config_file, 'w') { |f| f.puts config }
  end
  puts "Wrote yum configuration files for #{Pkg::Config.project} at #{Pkg::Config.ref} to pkg/#{target}/rpm"
end
repo_creation_command(repo_directory, artifact_paths = nil) click to toggle source
# File lib/packaging/rpm/repo.rb, line 25
def repo_creation_command(repo_directory, artifact_paths = nil)
  cmd = "[ -d #{repo_directory} ] || exit 1 ; "
  cmd << "pushd #{repo_directory} > /dev/null && "
  cmd << 'echo "Checking for running repo creation. Will wait if detected." && '
  cmd << 'while [ -f .lock ] ; do sleep 1 ; echo -n "." ; done && '
  cmd << 'echo "Setting lock" && '
  cmd << 'touch .lock && '
  cmd << 'createrepo=$(which createrepo) ; '

  # Added for compatibility.
  # The nightly repo ships operate differently and do not want to be calculating
  # the correct paths based on which packages are available on the distribution
  # host, we just want to be `createrepo`ing for what we've staged locally
  #
  # We should only assume repo_directory exists locally if we didn't pass
  # artifact paths
  if artifact_paths.nil?
    # Since the command already has a `pushd #{repo_directory}` let's make sure
    # we're calculating artifact paths relative to that.
    Dir.chdir repo_directory do
      artifact_paths = Dir.glob('**/*.rpm').map { |package| File.dirname(package) }
    end
  end

  artifact_paths.each do |path|
    next if path.include? 'aix'
    cmd << "if [ -d #{path}  ]; then "
    cmd << "pushd #{path} && "
    cmd << '$createrepo --checksum=sha --checkts --update --delta-workers=0 --database . && '
    cmd << 'popd ; '
    cmd << 'fi ;'
  end
  cmd
end
repo_deployment_command(origin_path, destination_path, destination, dryrun = false) click to toggle source

@deprecated this command will die a painful death when we are

able to sit down with Operations and refactor our distribution infra.
At a minimum, it should be refactored alongside its Debian counterpart
into something modestly more generic.
- Ryan McKern 11/2015

@param origin_path [String] path for RPM repos on local filesystem @param destination_path [String] path for RPM repos on remote filesystem @param destination [String] remote host to send rsynced content to. If

nil will copy locally

@param dryrun [Boolean] whether or not to use '–dry-run'

@return [String] an rsync command that can be executed on a remote host

to copy local content from that host to a remote node.
# File lib/packaging/rpm/repo.rb, line 74
def repo_deployment_command(origin_path, destination_path, destination, dryrun = false)
  path = Pathname.new(origin_path)
  dest_path = Pathname.new(destination_path)

  options = %w(
    rsync
    --recursive
    --links
    --hard-links
    --update
    --human-readable
    --itemize-changes
    --progress
    --verbose
    --super
    --delay-updates
    --omit-dir-times
    --no-perms
    --no-owner
    --no-group
  )

  options << '--dry-run' if dryrun
  options << path

  if destination
    options << "#{destination}:#{dest_path.parent}"
  else
    options << "#{dest_path.parent}"
  end

  options.join("\s")
end
retrieve_repo_configs(target = "repo_configs") click to toggle source
# File lib/packaging/rpm/repo.rb, line 115
def retrieve_repo_configs(target = "repo_configs")
  wget = Pkg::Util::Tool.check_tool("wget")
  FileUtils.mkdir_p("pkg/#{target}")
  config_url = "#{base_url}/#{target}/rpm/"
  begin
    stdout, _, _ = Pkg::Util::Execution.capture3("#{wget} -r -np -nH --cut-dirs 3 -P pkg/#{target} --reject 'index*' #{config_url}")
    stdout
  rescue => e
    fail "Couldn't retrieve rpm yum repo configs.\n#{e}"
  end
end
ship_repo_configs(target = "repo_configs") click to toggle source
# File lib/packaging/rpm/repo.rb, line 11
def ship_repo_configs(target = "repo_configs")
  if Pkg::Util::File.empty_dir?("pkg/#{target}/rpm")
    warn "No repo configs have been generated! Try pl:rpm_repo_configs."
    return
  end

  Pkg::Util::RakeUtils.invoke_task("pl:fetch")
  repo_dir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{target}/rpm"
  Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
  Pkg::Util::Execution.retry_on_fail(:times => 3) do
    Pkg::Util::Net.rsync_to("pkg/#{target}/rpm/", Pkg::Config.distribution_server, repo_dir)
  end
end
sign_repos(directory) click to toggle source
# File lib/packaging/rpm/repo.rb, line 108
def sign_repos(directory)
  files_to_sign = Find.find(directory).select { |file| file.match(/repomd.xml$/) }
  files_to_sign.each do |file|
    Pkg::Util::Gpg.sign_file(file)
  end
end