class Dependabot::Python::UpdateChecker::PipCompileVersionResolver

This class does version resolution for pip-compile. Its approach is:

rubocop:disable Metrics/ClassLength

Constants

GIT_DEPENDENCY_UNREACHABLE_REGEX
GIT_REFERENCE_NOT_FOUND_REGEX
NATIVE_COMPILATION_ERROR
VERBOSE_ERROR_OUTPUT_LINES

Attributes

credentials[R]
dependency[R]
dependency_files[R]

Public Class Methods

new(dependency:, dependency_files:, credentials:) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 36
def initialize(dependency:, dependency_files:, credentials:)
  @dependency               = dependency
  @dependency_files         = dependency_files
  @credentials              = credentials
  @build_isolation = true
end

Public Instance Methods

latest_resolvable_version(requirement: nil) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 43
def latest_resolvable_version(requirement: nil)
  version_string =
    fetch_latest_resolvable_version_string(requirement: requirement)

  version_string.nil? ? nil : Python::Version.new(version_string)
end
resolvable?(version:) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 50
def resolvable?(version:)
  @resolvable ||= {}
  return @resolvable[version] if @resolvable.key?(version)

  @resolvable[version] = if fetch_latest_resolvable_version_string(requirement: "==#{version}")
                           true
                         else
                           false
                         end
end

Private Instance Methods

check_original_requirements_resolvable() click to toggle source

Needed because pip-compile's resolver isn't perfect. Note: We raise errors from this method, rather than returning a boolean, so that all deps for this repo will raise identical errors when failing to update

# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 166
def check_original_requirements_resolvable
  SharedHelpers.in_a_temporary_directory do
    SharedHelpers.with_git_configured(credentials: credentials) do
      write_temporary_dependency_files(update_requirement: false)

      filenames_to_compile.each do |filename|
        run_pip_compile_command(
          "pyenv exec pip-compile #{pip_compile_options(filename)} --allow-unsafe #{filename}"
        )
      end

      true
    rescue SharedHelpers::HelperSubprocessFailed => e
      # Pick the error message that includes resolvability errors, this might be the cause from
      # handle_pip_compile_errors (it's unclear if we should always pick the cause here)
      error_message = [e.message, e.cause&.message].compact.find do |msg|
        ["UnsupportedConstraint", "Could not find a version"].any? { |err| msg.include?(err) }
      end

      cleaned_message = clean_error_message(error_message || "")
      raise if cleaned_message.empty?

      raise DependencyFileNotResolvable, cleaned_message
    end
  end
end
clean_error_message(message) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 355
def clean_error_message(message)
  msg_lines = message.lines
  msg = msg_lines.
        take_while { |l| !l.start_with?("During handling of") }.
        drop_while { |l| l.start_with?(*VERBOSE_ERROR_OUTPUT_LINES) }.
        join.strip

  # Redact any URLs, as they may include credentials
  msg.gsub(/http.*?(?=\s)/, "<redacted>")
end
compilation_error?(error) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 108
def compilation_error?(error)
  error.message.include?(NATIVE_COMPILATION_ERROR)
end
compiled_file_for_filename(filename) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 383
def compiled_file_for_filename(filename)
  compiled_file =
    compiled_files.
    find { |f| f.content.match?(output_file_regex(filename)) }

  compiled_file ||=
    compiled_files.
    find { |f| f.name == filename.gsub(/\.in$/, ".txt") }

  compiled_file
end
compiled_file_includes_dependency?(compiled_file) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 399
def compiled_file_includes_dependency?(compiled_file)
  return false unless compiled_file

  regex = RequirementParser::INSTALL_REQ_WITH_REQUIREMENT

  matches = []
  compiled_file.content.scan(regex) { matches << Regexp.last_match }
  matches.any? { |m| normalise(m[:name]) == dependency.name }
end
compiled_files() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 516
def compiled_files
  dependency_files.select { |f| f.name.end_with?(".txt") }
end
error_certainly_bad_python_version?(message) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 256
def error_certainly_bad_python_version?(message)
  return true if message.include?("UnsupportedPythonVersion")

  unless message.include?('"python setup.py egg_info" failed') ||
         message.include?("exit status 1: python setup.py egg_info")
    return false
  end

  message.include?("SyntaxError")
end
fetch_latest_resolvable_version_string(requirement:) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 63
def fetch_latest_resolvable_version_string(requirement:)
  @latest_resolvable_version_string ||= {}
  return @latest_resolvable_version_string[requirement] if @latest_resolvable_version_string.key?(requirement)

  @latest_resolvable_version_string[requirement] ||=
    SharedHelpers.in_a_temporary_directory do
      SharedHelpers.with_git_configured(credentials: credentials) do
        write_temporary_dependency_files(updated_req: requirement)
        install_required_python

        filenames_to_compile.each do |filename|
          # Shell out to pip-compile.
          # This is slow, as pip-compile needs to do installs.
          run_pip_compile_command(
            "pyenv exec pip-compile --allow-unsafe -v "\
             "#{pip_compile_options(filename)} -P #{dependency.name} "\
             "#{filename}"
          )
          # Run pip-compile a second time, without an update argument,
          # to ensure it handles markers correctly
          write_original_manifest_files unless dependency.top_level?
          run_pip_compile_command(
            "pyenv exec pip-compile --allow-unsafe "\
             "#{pip_compile_options(filename)} #{filename}"
          )
        end

        # Remove any .python-version file before parsing the reqs
        FileUtils.remove_entry(".python-version", true)

        parse_updated_files
      end
    rescue SharedHelpers::HelperSubprocessFailed => e
      retry_count ||= 0
      retry_count += 1

      if compilation_error?(e) && retry_count <= 1
        @build_isolation = false
        retry
      end

      handle_pip_compile_errors(e)
    end
end
filenames_to_compile() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 366
def filenames_to_compile
  files_from_reqs =
    dependency.requirements.
    map { |r| r[:file] }.
    select { |fn| fn.end_with?(".in") }

  files_from_compiled_files =
    pip_compile_files.map(&:name).select do |fn|
      compiled_file = compiled_file_for_filename(fn)
      compiled_file_includes_dependency?(compiled_file)
    end

  filenames = [*files_from_reqs, *files_from_compiled_files].uniq

  order_filenames_for_compilation(filenames)
end
handle_pip_compile_errors(error) click to toggle source

rubocop:disable Metrics/AbcSize

# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 113
def handle_pip_compile_errors(error)
  if error.message.include?("Could not find a version")
    check_original_requirements_resolvable
    # If the original requirements are resolvable but we get an
    # incompatibility error after unlocking then it's likely to be
    # due to problems with pip-compile's cascading resolution
    return nil
  end

  if error.message.include?("UnsupportedConstraint")
    # If there's an unsupported constraint, check if it existed
    # previously (and raise if it did)
    check_original_requirements_resolvable
  end

  if (error.message.include?('Command "python setup.py egg_info') ||
      error.message.include?(
        "exit status 1: python setup.py egg_info"
      )) &&
     check_original_requirements_resolvable
    # The latest version of the dependency we're updating is borked
    # (because it has an unevaluatable setup.py). Skip the update.
    return
  end

  if error.message.include?("Could not find a version ") &&
     !error.message.match?(/#{Regexp.quote(dependency.name)}/i)
    # Sometimes pip-tools gets confused and can't work around
    # sub-dependency incompatibilities. Ignore those cases.
    return nil
  end

  if error.message.match?(GIT_REFERENCE_NOT_FOUND_REGEX)
    name = error.message.match(GIT_REFERENCE_NOT_FOUND_REGEX).
           named_captures.fetch("name")
    raise GitDependencyReferenceNotFound, name
  end

  if error.message.match?(GIT_DEPENDENCY_UNREACHABLE_REGEX)
    url = error.message.match(GIT_DEPENDENCY_UNREACHABLE_REGEX).
          named_captures.fetch("url")
    raise GitDependenciesNotReachable, url
  end

  raise
end
install_required_python() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 302
def install_required_python
  return if run_command("pyenv versions").include?("#{python_version}\n")

  run_command("pyenv install -s #{python_version}")
  run_command("pyenv exec pip install -r"\
              "#{NativeHelpers.python_requirements_path}")
end
normalise(name) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 341
def normalise(name)
  NameNormaliser.normalise(name)
end
order_filenames_for_compilation(filenames) click to toggle source

If the files we need to update require one another then we need to update them in the right order

# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 411
def order_filenames_for_compilation(filenames)
  ordered_filenames = []

  while (remaining_filenames = filenames - ordered_filenames).any?
    ordered_filenames +=
      remaining_filenames.
      select do |fn|
        unupdated_reqs = requirement_map[fn] - ordered_filenames
        (unupdated_reqs & filenames).empty?
      end
  end

  ordered_filenames
end
output_file_regex(filename) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 395
def output_file_regex(filename)
  "--output-file[=\s]+.*\s#{Regexp.escape(filename)}\s*$"
end
parse_updated_files() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 445
def parse_updated_files
  updated_files =
    dependency_files.map do |file|
      next file if file.name == ".python-version"

      updated_file = file.dup
      updated_file.content = File.read(file.name)
      updated_file
    end

  Python::FileParser.new(
    dependency_files: updated_files,
    source: nil,
    credentials: credentials
  ).parse.find { |d| d.name == dependency.name }&.version
end
pip_compile_files() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 512
def pip_compile_files
  dependency_files.select { |f| f.name.end_with?(".in") }
end
pip_compile_index_options() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 222
def pip_compile_index_options
  credentials.
    select { |cred| cred["type"] == "python_index" }.
    map do |cred|
      authed_url = AuthedUrlBuilder.authed_url(credential: cred)

      if cred["replaces-base"]
        "--index-url=#{authed_url}"
      else
        "--extra-index-url=#{authed_url}"
      end
    end
end
pip_compile_options(filename) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 211
def pip_compile_options(filename)
  options = @build_isolation ? ["--build-isolation"] : ["--no-build-isolation"]
  options += pip_compile_index_options

  if (requirements_file = compiled_file_for_filename(filename))
    options << "--output-file=#{requirements_file.name}"
  end

  options.join(" ")
end
pre_installed_python?(version) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 504
def pre_installed_python?(version)
  PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.include?(version)
end
python_env() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 241
def python_env
  env = {}

  # Handle Apache Airflow 1.10.x installs
  if dependency_files.any? { |f| f.content.include?("apache-airflow") }
    if dependency_files.any? { |f| f.content.include?("unidecode") }
      env["AIRFLOW_GPL_UNIDECODE"] = "yes"
    else
      env["SLUGIFY_USES_TEXT_UNIDECODE"] = "yes"
    end
  end

  env
end
python_requirement_parser() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 497
def python_requirement_parser
  @python_requirement_parser ||=
    FileParser::PythonRequirementParser.new(
      dependency_files: dependency_files
    )
end
python_version() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 462
def python_version
  @python_version ||=
    user_specified_python_version ||
    python_version_matching_imputed_requirements ||
    PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.first
end
python_version_matching(requirements) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 486
def python_version_matching(requirements)
  PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.find do |version_string|
    version = Python::Version.new(version_string)
    requirements.all? do |req|
      next req.any? { |r| r.satisfied_by?(version) } if req.is_a?(Array)

      req.satisfied_by?(version)
    end
  end
end
python_version_matching_imputed_requirements() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 478
def python_version_matching_imputed_requirements
  compiled_file_python_requirement_markers =
    python_requirement_parser.imputed_requirements.map do |r|
      Dependabot::Python::Requirement.new(r)
    end
  python_version_matching(compiled_file_python_requirement_markers)
end
requirement_map() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 426
def requirement_map
  child_req_regex = Python::FileFetcher::CHILD_REQUIREMENT_REGEX
  @requirement_map ||=
    pip_compile_files.each_with_object({}) do |file, req_map|
      paths = file.content.scan(child_req_regex).flatten
      current_dir = File.dirname(file.name)

      req_map[file.name] =
        paths.map do |path|
          path = File.join(current_dir, path) if current_dir != "."
          path = Pathname.new(path).cleanpath.to_path
          path = path.gsub(/\.txt$/, ".in")
          next if path == file.name

          path
        end.uniq.compact
    end
end
run_command(command, env: python_env) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 193
def run_command(command, env: python_env)
  start = Time.now
  command = SharedHelpers.escape_command(command)
  stdout, process = Open3.capture2e(env, command)
  time_taken = Time.now - start

  return stdout if process.success?

  raise SharedHelpers::HelperSubprocessFailed.new(
    message: stdout,
    error_context: {
      command: command,
      time_taken: time_taken,
      process_exit_value: process.to_s
    }
  )
end
run_pip_compile_command(command) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 236
def run_pip_compile_command(command)
  run_command("pyenv local #{python_version}")
  run_command(command)
end
sanitized_setup_file_content(file) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 310
def sanitized_setup_file_content(file)
  @sanitized_setup_file_content ||= {}
  return @sanitized_setup_file_content[file.name] if @sanitized_setup_file_content[file.name]

  @sanitized_setup_file_content[file.name] =
    Python::FileUpdater::SetupFileSanitizer.
    new(setup_file: file, setup_cfg: setup_cfg(file)).
    sanitized_content
end
setup_cfg(file) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 320
def setup_cfg(file)
  dependency_files.find do |f|
    f.name == file.name.sub(/\.py$/, ".cfg")
  end
end
setup_cfg_files() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 520
def setup_cfg_files
  dependency_files.select { |f| f.name.end_with?("setup.cfg") }
end
setup_files() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 508
def setup_files
  dependency_files.select { |f| f.name.end_with?("setup.py") }
end
update_req_file(file, updated_req) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 326
def update_req_file(file, updated_req)
  return file.content unless file.name.end_with?(".in")

  req = dependency.requirements.find { |r| r[:file] == file.name }

  return file.content + "\n#{dependency.name} #{updated_req}" unless req&.fetch(:requirement)

  Python::FileUpdater::RequirementReplacer.new(
    content: file.content,
    dependency_name: dependency.name,
    old_requirement: req[:requirement],
    new_requirement: updated_req
  ).updated_content
end
user_specified_python_version() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 469
def user_specified_python_version
  return unless python_requirement_parser.user_specified_requirements.any?

  user_specified_requirements =
    python_requirement_parser.user_specified_requirements.
    map { |r| Python::Requirement.requirements_array(r) }
  python_version_matching(user_specified_requirements)
end
write_original_manifest_files() click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 295
def write_original_manifest_files
  pip_compile_files.each do |file|
    FileUtils.mkdir_p(Pathname.new(file.name).dirname)
    File.write(file.name, file.content)
  end
end
write_temporary_dependency_files(updated_req: nil, update_requirement: true) click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 267
def write_temporary_dependency_files(updated_req: nil,
                                     update_requirement: true)
  dependency_files.each do |file|
    path = file.name
    FileUtils.mkdir_p(Pathname.new(path).dirname)
    updated_content =
      if update_requirement then update_req_file(file, updated_req)
      else file.content
      end
    File.write(path, updated_content)
  end

  # Overwrite the .python-version with updated content
  File.write(".python-version", python_version)

  setup_files.each do |file|
    path = file.name
    FileUtils.mkdir_p(Pathname.new(path).dirname)
    File.write(path, sanitized_setup_file_content(file))
  end

  setup_cfg_files.each do |file|
    path = file.name
    FileUtils.mkdir_p(Pathname.new(path).dirname)
    File.write(path, "[metadata]\nname = sanitized-package\n")
  end
end