class Dependabot::Python::FileUpdater::PipCompileFileUpdater

rubocop:disable Metrics/ClassLength

Constants

INCOMPATIBLE_VERSIONS_REGEX
UNSAFE_NOTE
UNSAFE_PACKAGES
WARNINGS

Attributes

credentials[R]
dependencies[R]
dependency_files[R]

Public Class Methods

new(dependencies:, dependency_files:, credentials:) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 32
def initialize(dependencies:, dependency_files:, credentials:)
  @dependencies = dependencies
  @dependency_files = dependency_files
  @credentials = credentials
end

Public Instance Methods

updated_dependency_files() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 38
def updated_dependency_files
  return @updated_dependency_files if @update_already_attempted

  @update_already_attempted = true
  @updated_dependency_files ||= fetch_updated_dependency_files
end

Private Instance Methods

compile_new_requirement_files() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 66
def compile_new_requirement_files
  SharedHelpers.in_a_temporary_directory do
    write_updated_dependency_files
    install_required_python

    filenames_to_compile.each do |filename|
      # Shell out to pip-compile, generate a new set of requirements.
      # This is slow, as pip-compile needs to do installs.
      name_part = "pyenv exec pip-compile "\
                  "#{pip_compile_options(filename)} -P "\
                  "#{dependency.name}"
      version_part = "#{dependency.version} #{filename}"
      # Don't escape pyenv `dep-name==version` syntax
      run_pip_compile_command(
        "#{SharedHelpers.escape_command(name_part)}=="\
        "#{SharedHelpers.escape_command(version_part)}",
        allow_unsafe_shell_command: true
      )
      # Run pip-compile a second time, without an update argument, to
      # ensure it resets the right comments.
      run_pip_compile_command(
        "pyenv exec pip-compile #{pip_compile_options(filename)} "\
        "#{filename}"
      )
    end

    # Remove any .python-version file before parsing the reqs
    FileUtils.remove_entry(".python-version", true)

    dependency_files.map do |file|
      next unless file.name.end_with?(".txt")

      updated_content = File.read(file.name)

      updated_content =
        post_process_compiled_file(updated_content, file)
      next if updated_content == file.content

      file.dup.tap { |f| f.content = updated_content }
    end.compact
  end
end
compiled_file_for_filename(filename) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 471
def compiled_file_for_filename(filename)
  compiled_file =
    compiled_files.
    find { |f| f.content.match?(output_file_regex(filename)) }

  compiled_file ||=
    compiled_files.
    find { |f| f.name == filename.gsub(/\.in$/, ".txt") }

  compiled_file
end
compiled_file_includes_dependency?(compiled_file) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 487
def compiled_file_includes_dependency?(compiled_file)
  return false unless compiled_file

  regex = RequirementParser::INSTALL_REQ_WITH_REQUIREMENT

  matches = []
  compiled_file.content.scan(regex) { matches << Regexp.last_match }
  matches.any? { |m| normalise(m[:name]) == dependency.name }
end
compiled_files() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 591
def compiled_files
  dependency_files.select { |f| f.name.end_with?(".txt") }
end
dependency() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 47
def dependency
  # For now, we'll only ever be updating a single dependency
  dependencies.first
end
deps_to_augment_hashes_for(updated_content, original_content) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 353
def deps_to_augment_hashes_for(updated_content, original_content)
  regex = /^#{RequirementParser::INSTALL_REQ_WITH_REQUIREMENT}/

  new_matches = []
  updated_content.scan(regex) { new_matches << Regexp.last_match }

  old_matches = []
  original_content.scan(regex) { old_matches << Regexp.last_match }

  new_deps = []
  changed_hashes_deps = []

  new_matches.each do |mtch|
    nm = mtch.named_captures["name"]
    old_match = old_matches.find { |m| m.named_captures["name"] == nm }

    next new_deps << mtch unless old_match
    next unless old_match.named_captures["hashes"]

    old_count = old_match.named_captures["hashes"].split("--hash").count
    new_count = mtch.named_captures["hashes"].split("--hash").count
    changed_hashes_deps << mtch if new_count < old_count
  end

  return [] if changed_hashes_deps.none?

  [*new_deps, *changed_hashes_deps]
end
fetch_updated_dependency_files() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 52
def fetch_updated_dependency_files
  updated_compiled_files = compile_new_requirement_files
  updated_manifest_files = update_manifest_files

  updated_files = updated_compiled_files + updated_manifest_files
  updated_uncompiled_files = update_uncompiled_files(updated_files)

  [
    *updated_manifest_files,
    *updated_compiled_files,
    *updated_uncompiled_files
  ]
end
filenames_to_compile() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 454
def filenames_to_compile
  files_from_reqs =
    dependency.requirements.
    map { |r| r[:file] }.
    select { |fn| fn.end_with?(".in") }

  files_from_compiled_files =
    pip_compile_files.map(&:name).select do |fn|
      compiled_file = compiled_file_for_filename(fn)
      compiled_file_includes_dependency?(compiled_file)
    end

  filenames = [*files_from_reqs, *files_from_compiled_files].uniq

  order_filenames_for_compilation(filenames)
end
freeze_dependency_requirement(file) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 246
def freeze_dependency_requirement(file)
  return file.content unless file.name.end_with?(".in")

  old_req = dependency.previous_requirements.
            find { |r| r[:file] == file.name }

  return file.content unless old_req
  return file.content if old_req == "==#{dependency.version}"

  RequirementReplacer.new(
    content: file.content,
    dependency_name: dependency.name,
    old_requirement: old_req[:requirement],
    new_requirement: "==#{dependency.version}"
  ).updated_content
end
handle_pip_errors(stdout, command, time_taken, exit_value) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 161
def handle_pip_errors(stdout, command, time_taken, exit_value)
  if stdout.match?(INCOMPATIBLE_VERSIONS_REGEX)
    raise DependencyFileNotResolvable, stdout.match(INCOMPATIBLE_VERSIONS_REGEX)
  end

  raise SharedHelpers::HelperSubprocessFailed.new(
    message: stdout,
    error_context: {
      command: command,
      time_taken: time_taken,
      process_exit_value: exit_value
    }
  )
end
hash_separator(requirement_string) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 390
def hash_separator(requirement_string)
  hash_regex = RequirementParser::HASH
  return unless requirement_string.match?(hash_regex)

  current_separator =
    requirement_string.
    match(/#{hash_regex}((?<separator>\s*\\?\s*?)#{hash_regex})*/).
    named_captures.fetch("separator")

  default_separator =
    requirement_string.
    match(RequirementParser::HASH).
    pre_match.match(/(?<separator>\s*\\?\s*?)\z/).
    named_captures.fetch("separator")

  current_separator || default_separator
end
includes_unsafe_packages?(content) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 450
def includes_unsafe_packages?(content)
  UNSAFE_PACKAGES.any? { |n| content.match?(/^#{Regexp.quote(n)}==/) }
end
install_required_python() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 222
def install_required_python
  return if run_command("pyenv versions").include?("#{python_version}\n")

  run_command("pyenv install -s #{python_version}")
  run_command("pyenv exec pip install -r "\
              "#{NativeHelpers.python_requirements_path}")
end
normalise(name) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 497
def normalise(name)
  NameNormaliser.normalise(name)
end
order_filenames_for_compilation(filenames) click to toggle source

If the files we need to update require one another then we need to update them in the right order

# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 503
def order_filenames_for_compilation(filenames)
  ordered_filenames = []

  while (remaining_filenames = filenames - ordered_filenames).any?
    ordered_filenames +=
      remaining_filenames.
      select do |fn|
        unupdated_reqs = requirement_map[fn] - ordered_filenames
        (unupdated_reqs & filenames).empty?
      end
  end

  ordered_filenames
end
output_file_regex(filename) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 483
def output_file_regex(filename)
  "--output-file[=\s]+.*\s#{Regexp.escape(filename)}\s*$"
end
package_hashes_for(name:, version:, algorithm:) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 382
def package_hashes_for(name:, version:, algorithm:)
  SharedHelpers.run_helper_subprocess(
    command: "pyenv exec python #{NativeHelpers.python_helper_path}",
    function: "get_dependency_hash",
    args: [name, version, algorithm]
  ).map { |h| "--hash=#{algorithm}:#{h['hash']}" }
end
pip_compile_files() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 587
def pip_compile_files
  dependency_files.select { |f| f.name.end_with?(".in") }
end
pip_compile_index_options() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 436
def pip_compile_index_options
  credentials.
    select { |cred| cred["type"] == "python_index" }.
    map do |cred|
      authed_url = AuthedUrlBuilder.authed_url(credential: cred)

      if cred["replaces-base"]
        "--index-url=#{authed_url}"
      else
        "--extra-index-url=#{authed_url}"
      end
    end
end
pip_compile_options(filename) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 408
def pip_compile_options(filename)
  options = ["--build-isolation"]
  options += pip_compile_index_options

  if (requirements_file = compiled_file_for_filename(filename))
    options += pip_compile_options_from_compiled_file(requirements_file)
  end

  options.join(" ")
end
pip_compile_options_from_compiled_file(requirements_file) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 419
def pip_compile_options_from_compiled_file(requirements_file)
  options = ["--output-file=#{requirements_file.name}"]

  options << "--no-emit-index-url" unless requirements_file.content.include?("index-url http")

  options << "--generate-hashes" if requirements_file.content.include?("--hash=sha")

  options << "--allow-unsafe" if includes_unsafe_packages?(requirements_file.content)

  options << "--no-annotate" unless requirements_file.content.include?("# via ")

  options << "--no-header" unless requirements_file.content.include?("autogenerated by pip-c")

  options << "--pre" if requirements_file.content.include?("--pre")
  options
end
post_process_compiled_file(updated_content, file) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 281
def post_process_compiled_file(updated_content, file)
  content = replace_header_with_original(updated_content, file.content)
  content = remove_new_warnings(content, file.content)
  content = update_hashes_if_required(content, file.content)
  replace_absolute_file_paths(content, file.content)
end
pre_installed_python?(version) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 579
def pre_installed_python?(version)
  PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.include?(version)
end
python_env() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 184
def python_env
  env = {}

  # Handle Apache Airflow 1.10.x installs
  if dependency_files.any? { |f| f.content.include?("apache-airflow") }
    if dependency_files.any? { |f| f.content.include?("unidecode") }
      env["AIRFLOW_GPL_UNIDECODE"] = "yes"
    else
      env["SLUGIFY_USES_TEXT_UNIDECODE"] = "yes"
    end
  end

  env
end
python_requirement_parser() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 572
def python_requirement_parser
  @python_requirement_parser ||=
    FileParser::PythonRequirementParser.new(
      dependency_files: dependency_files
    )
end
python_version() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 537
def python_version
  @python_version ||=
    user_specified_python_version ||
    python_version_matching_imputed_requirements ||
    PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.first
end
python_version_matching(requirements) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 561
def python_version_matching(requirements)
  PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.find do |version_string|
    version = Python::Version.new(version_string)
    requirements.all? do |req|
      next req.any? { |r| r.satisfied_by?(version) } if req.is_a?(Array)

      req.satisfied_by?(version)
    end
  end
end
python_version_matching_imputed_requirements() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 553
def python_version_matching_imputed_requirements
  compiled_file_python_requirement_markers =
    python_requirement_parser.imputed_requirements.map do |r|
      Dependabot::Python::Requirement.new(r)
    end
  python_version_matching(compiled_file_python_requirement_markers)
end
remove_new_warnings(updated_content, original_content) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 319
def remove_new_warnings(updated_content, original_content)
  content = updated_content

  content = content.sub(WARNINGS, "\n") if content.match?(WARNINGS) && !original_content.match?(WARNINGS)

  if content.match?(UNSAFE_NOTE) &&
     !original_content.match?(UNSAFE_NOTE)
    content = content.sub(UNSAFE_NOTE, "\n")
  end

  content
end
replace_absolute_file_paths(updated_content, original_content) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 298
def replace_absolute_file_paths(updated_content, original_content)
  content = updated_content

  update_count = 0
  original_content.lines.each do |original_line|
    next unless original_line.start_with?("-e")
    next update_count += 1 if updated_content.include?(original_line)

    line_to_update =
      updated_content.lines.
      select { |l| l.start_with?("-e") }.
      at(update_count)
    raise "Mismatch in editable requirements!" unless line_to_update

    content = content.gsub(line_to_update, original_line)
    update_count += 1
  end

  content
end
replace_header_with_original(updated_content, original_content) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 288
def replace_header_with_original(updated_content, original_content)
  original_header_lines =
    original_content.lines.take_while { |l| l.start_with?("#") }

  updated_content_lines =
    updated_content.lines.drop_while { |l| l.start_with?("#") }

  [*original_header_lines, *updated_content_lines].join
end
requirement_map() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 518
def requirement_map
  child_req_regex = Python::FileFetcher::CHILD_REQUIREMENT_REGEX
  @requirement_map ||=
    pip_compile_files.each_with_object({}) do |file, req_map|
      paths = file.content.scan(child_req_regex).flatten
      current_dir = File.dirname(file.name)

      req_map[file.name] =
        paths.map do |path|
          path = File.join(current_dir, path) if current_dir != "."
          path = Pathname.new(path).cleanpath.to_path
          path = path.gsub(/\.txt$/, ".in")
          next if path == file.name

          path
        end.uniq.compact
    end
end
run_command(cmd, env: python_env, allow_unsafe_shell_command: false) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 146
def run_command(cmd, env: python_env, allow_unsafe_shell_command: false)
  start = Time.now
  command = if allow_unsafe_shell_command
              cmd
            else
              SharedHelpers.escape_command(cmd)
            end
  stdout, process = Open3.capture2e(env, command)
  time_taken = Time.now - start

  return stdout if process.success?

  handle_pip_errors(stdout, command, time_taken, process.to_s)
end
run_pip_compile_command(command, allow_unsafe_shell_command: false) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 176
def run_pip_compile_command(command, allow_unsafe_shell_command: false)
  run_command("pyenv local #{python_version}")
  run_command(
    command,
    allow_unsafe_shell_command: allow_unsafe_shell_command
  )
end
sanitized_setup_file_content(file) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 230
def sanitized_setup_file_content(file)
  @sanitized_setup_file_content ||= {}
  return @sanitized_setup_file_content[file.name] if @sanitized_setup_file_content[file.name]

  @sanitized_setup_file_content[file.name] =
    SetupFileSanitizer.
    new(setup_file: file, setup_cfg: setup_cfg(file)).
    sanitized_content
end
setup_cfg(file) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 240
def setup_cfg(file)
  dependency_files.find do |f|
    f.name == file.name.sub(/\.py$/, ".cfg")
  end
end
setup_cfg_files() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 595
def setup_cfg_files
  dependency_files.select { |f| f.name.end_with?("setup.cfg") }
end
setup_files() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 583
def setup_files
  dependency_files.select { |f| f.name.end_with?("setup.py") }
end
update_dependency_requirement(file) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 263
def update_dependency_requirement(file)
  return file.content unless file.name.end_with?(".in")

  old_req = dependency.previous_requirements.
            find { |r| r[:file] == file.name }
  new_req = dependency.requirements.
            find { |r| r[:file] == file.name }
  return file.content unless old_req&.fetch(:requirement)
  return file.content if old_req == new_req

  RequirementReplacer.new(
    content: file.content,
    dependency_name: dependency.name,
    old_requirement: old_req[:requirement],
    new_requirement: new_req[:requirement]
  ).updated_content
end
update_hashes_if_required(updated_content, original_content) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 332
def update_hashes_if_required(updated_content, original_content)
  deps_to_update =
    deps_to_augment_hashes_for(updated_content, original_content)

  updated_content_with_hashes = updated_content
  deps_to_update.each do |mtch|
    updated_string = mtch.to_s.sub(
      RequirementParser::HASHES,
      package_hashes_for(
        name: mtch.named_captures.fetch("name"),
        version: mtch.named_captures.fetch("version"),
        algorithm: mtch.named_captures.fetch("algorithm")
      ).sort.join(hash_separator(mtch.to_s))
    )

    updated_content_with_hashes = updated_content_with_hashes.
                                  gsub(mtch.to_s, updated_string)
  end
  updated_content_with_hashes
end
update_manifest_files() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 109
def update_manifest_files
  dependency_files.map do |file|
    next unless file.name.end_with?(".in")

    file = file.dup
    updated_content = update_dependency_requirement(file)
    next if updated_content == file.content

    file.content = updated_content
    file
  end.compact
end
update_uncompiled_files(updated_files) click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 122
def update_uncompiled_files(updated_files)
  updated_filenames = updated_files.map(&:name)
  old_reqs = dependency.previous_requirements.
             reject { |r| updated_filenames.include?(r[:file]) }
  new_reqs = dependency.requirements.
             reject { |r| updated_filenames.include?(r[:file]) }

  return [] if new_reqs.none?

  files = dependency_files.
          reject { |file| updated_filenames.include?(file.name) }

  args = dependency.to_h
  args = args.keys.map { |k| [k.to_sym, args[k]] }.to_h
  args[:requirements] = new_reqs
  args[:previous_requirements] = old_reqs

  RequirementFileUpdater.new(
    dependencies: [Dependency.new(**args)],
    dependency_files: files,
    credentials: credentials
  ).updated_dependency_files
end
user_specified_python_version() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 544
def user_specified_python_version
  return unless python_requirement_parser.user_specified_requirements.any?

  user_specified_requirements =
    python_requirement_parser.user_specified_requirements.
    map { |r| Python::Requirement.requirements_array(r) }
  python_version_matching(user_specified_requirements)
end
write_updated_dependency_files() click to toggle source
# File lib/dependabot/python/file_updater/pip_compile_file_updater.rb, line 199
def write_updated_dependency_files
  dependency_files.each do |file|
    path = file.name
    FileUtils.mkdir_p(Pathname.new(path).dirname)
    File.write(path, freeze_dependency_requirement(file))
  end

  # Overwrite the .python-version with updated content
  File.write(".python-version", python_version)

  setup_files.each do |file|
    path = file.name
    FileUtils.mkdir_p(Pathname.new(path).dirname)
    File.write(path, sanitized_setup_file_content(file))
  end

  setup_cfg_files.each do |file|
    path = file.name
    FileUtils.mkdir_p(Pathname.new(path).dirname)
    File.write(path, "[metadata]\nname = sanitized-package\n")
  end
end