class Dependabot::Python::UpdateChecker::PipenvVersionResolver

This class does version resolution for Pipfiles. Its current approach is somewhat crude:

Unfortunately, Pipenv doesn't resolve how we'd expect - it appears to just raise if the latest version can't be resolved. Knowing that is still better than nothing, though.

Constants

GIT_DEPENDENCY_UNREACHABLE_REGEX
GIT_REFERENCE_NOT_FOUND_REGEX
PIPENV_INSTALLATION_ERROR
PIPENV_INSTALLATION_ERROR_REGEX
PIPENV_RANGE_WARNING
TRACEBACK
UNSUPPORTED_DEP_REGEX

Attributes

credentials[R]
dependency[R]
dependency_files[R]

Public Class Methods

new(dependency:, dependency_files:, credentials:) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 48
def initialize(dependency:, dependency_files:, credentials:)
  @dependency               = dependency
  @dependency_files         = dependency_files
  @credentials              = credentials
end

Public Instance Methods

latest_resolvable_version(requirement: nil) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 54
def latest_resolvable_version(requirement: nil)
  version_string =
    fetch_latest_resolvable_version_string(requirement: requirement)

  version_string.nil? ? nil : Python::Version.new(version_string)
end
resolvable?(version:) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 61
def resolvable?(version:)
  @resolvable ||= {}
  return @resolvable[version] if @resolvable.key?(version)

  @resolvable[version] = if fetch_latest_resolvable_version_string(requirement: "==#{version}")
                           true
                         else
                           false
                         end
end

Private Instance Methods

add_private_sources(pipfile_content) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 391
def add_private_sources(pipfile_content)
  Python::FileUpdater::PipfilePreparer.
    new(pipfile_content: pipfile_content).
    replace_sources(credentials)
end
check_original_requirements_resolvable() click to toggle source

Needed because Pipenv's resolver isn't perfect. Note: We raise errors from this method, rather than returning a boolean, so that all deps for this repo will raise identical errors when failing to update

# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 208
def check_original_requirements_resolvable
  SharedHelpers.in_a_temporary_directory do
    SharedHelpers.with_git_configured(credentials: credentials) do
      write_temporary_dependency_files(update_pipfile: false)

      run_pipenv_command("pyenv exec pipenv lock")

      true
    rescue SharedHelpers::HelperSubprocessFailed => e
      handle_pipenv_errors_resolving_original_reqs(e)
    end
  end
end
clean_error_message(message) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 249
def clean_error_message(message)
  # Pipenv outputs a lot of things to STDERR, so we need to clean
  # up the error message
  msg_lines = message.lines
  msg = msg_lines.
        take_while { |l| !l.start_with?("During handling of") }.
        drop_while do |l|
          next false if l.start_with?("CRITICAL:")
          next false if l.start_with?("ERROR:")
          next false if l.start_with?("packaging.specifiers")
          next false if l.start_with?("pipenv.patched.notpip._internal")
          next false if l.include?("Max retries exceeded")

          true
        end.join.strip

  # We also need to redact any URLs, as they may include credentials
  msg.gsub(/http.*?(?=\s)/, "<redacted>")
end
fetch_latest_resolvable_version_string(requirement:) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 74
def fetch_latest_resolvable_version_string(requirement:)
  @latest_resolvable_version_string ||= {}
  return @latest_resolvable_version_string[requirement] if @latest_resolvable_version_string.key?(requirement)

  @latest_resolvable_version_string[requirement] ||=
    SharedHelpers.in_a_temporary_directory do
      SharedHelpers.with_git_configured(credentials: credentials) do
        write_temporary_dependency_files(updated_req: requirement)
        install_required_python

        # Shell out to Pipenv, which handles everything for us.
        # Whilst calling `lock` avoids doing an install as part of the
        # pipenv flow, an install is still done by pip-tools in order
        # to resolve the dependencies. That means this is slow.
        run_pipenv_command("pyenv exec pipenv lock")

        updated_lockfile = JSON.parse(File.read("Pipfile.lock"))

        fetch_version_from_parsed_lockfile(updated_lockfile)
      end
    rescue SharedHelpers::HelperSubprocessFailed => e
      handle_pipenv_errors(e)
    end
end
fetch_version_from_parsed_lockfile(updated_lockfile) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 99
def fetch_version_from_parsed_lockfile(updated_lockfile)
  if dependency.requirements.any?
    group = dependency.requirements.first[:groups].first
    deps = updated_lockfile[group] || {}

    version =
      deps.transform_keys { |k| normalise(k) }.
      dig(dependency.name, "version")&.
      gsub(/^==/, "")

    return version
  end

  Python::FileParser::DEPENDENCY_GROUP_KEYS.each do |keys|
    deps = updated_lockfile[keys.fetch(:lockfile)] || {}
    version =
      deps.transform_keys { |k| normalise(k) }.
      dig(dependency.name, "version")&.
      gsub(/^==/, "")

    return version if version
  end

  # If the sub-dependency no longer appears in the lockfile return nil
  nil
end
freeze_other_dependencies(pipfile_content) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 350
def freeze_other_dependencies(pipfile_content)
  Python::FileUpdater::PipfilePreparer.
    new(pipfile_content: pipfile_content, lockfile: lockfile).
    freeze_top_level_dependencies_except([dependency])
end
handle_pipenv_errors(error) click to toggle source

rubocop:disable Metrics/CyclomaticComplexity rubocop:disable Metrics/PerceivedComplexity rubocop:disable Metrics/AbcSize rubocop:disable Metrics/MethodLength

# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 130
def handle_pipenv_errors(error)
  if error.message.include?("no version found at all") ||
     error.message.include?("Invalid specifier:") ||
     error.message.include?("Max retries exceeded")
    msg = clean_error_message(error.message)
    raise if msg.empty?

    raise DependencyFileNotResolvable, msg
  end

  if error.message.match?(UNSUPPORTED_DEP_REGEX)
    msg = "Dependabot detected a dependency that can't be built on "\
          "linux. Currently, all Dependabot builds happen on linux "\
          "boxes, so there is no way for Dependabot to resolve your "\
          "dependency files.\n\n"\
          "Unless you think Dependabot has made a mistake (please "\
          "tag us if so) you may wish to disable Dependabot on this "\
          "repo."
    raise DependencyFileNotResolvable, msg
  end

  if error.message.match?(PIPENV_RANGE_WARNING)
    msg = "Pipenv does not support specifying Python ranges "\
      "(see https://github.com/pypa/pipenv/issues/1050 for more "\
      "details)."
    raise DependencyFileNotResolvable, msg
  end

  check_original_requirements_resolvable if error.message.include?("Could not find a version")

  if error.message.include?("SyntaxError: invalid syntax")
    raise DependencyFileNotResolvable,
          "SyntaxError while installing dependencies. Is one of the dependencies not Python 3 compatible? "\
          "Pip v21 no longer supports Python 2."
  end

  if (error.message.include?('Command "python setup.py egg_info"') ||
      error.message.include?(
        "exit status 1: python setup.py egg_info"
      )) &&
     check_original_requirements_resolvable
    # The latest version of the dependency we're updating is borked
    # (because it has an unevaluatable setup.py). Skip the update.
    return
  end

  if error.message.include?("UnsupportedPythonVersion") &&
     user_specified_python_requirement
    check_original_requirements_resolvable

    # The latest version of the dependency we're updating to needs a
    # different Python version. Skip the update.
    return if error.message.match?(/#{Regexp.quote(dependency.name)}/i)
  end

  if error.message.match?(GIT_DEPENDENCY_UNREACHABLE_REGEX)
    url = error.message.match(GIT_DEPENDENCY_UNREACHABLE_REGEX).
          named_captures.fetch("url")
    raise GitDependenciesNotReachable, url
  end

  if error.message.match?(GIT_REFERENCE_NOT_FOUND_REGEX)
    name = error.message.match(GIT_REFERENCE_NOT_FOUND_REGEX).
           named_captures.fetch("name")
    raise GitDependencyReferenceNotFound, name
  end

  raise unless error.message.include?("could not be resolved")
end
handle_pipenv_errors_resolving_original_reqs(error) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 222
def handle_pipenv_errors_resolving_original_reqs(error)
  if error.message.include?("Could not find a version")
    msg = clean_error_message(error.message)
    msg.gsub!(/\s+\(from .*$/, "")
    raise if msg.empty?

    raise DependencyFileNotResolvable, msg
  end

  if error.message.include?("UnsupportedPythonVersion") &&
     user_specified_python_requirement
    msg = clean_error_message(error.message).
          lines.take_while { |l| !l.start_with?("File") }.join.strip
    raise if msg.empty?

    raise DependencyFileNotResolvable, msg
  end

  # NOTE: Pipenv masks the actualy error, see this issue for updates:
  # https://github.com/pypa/pipenv/issues/2791
  handle_pipenv_installation_error(error.message) if error.message.match?(PIPENV_INSTALLATION_ERROR_REGEX)

  # Raise an unhandled error, as this could be a problem with
  # Dependabot's infrastructure, rather than the Pipfile
  raise
end
handle_pipenv_installation_error(error_message) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 269
def handle_pipenv_installation_error(error_message)
  # Find the dependency that's causing resolution to fail
  dependency_name = error_message.match(PIPENV_INSTALLATION_ERROR_REGEX).named_captures["name"]
  raise unless dependency_name

  msg = "Pipenv failed to install \"#{dependency_name}\". This could be caused by missing system "\
        "dependencies that can't be installed by Dependabot or required installation flags.\n\n"\
        "Error output from running \"pipenv lock\":\n"\
        "#{clean_error_message(error_message)}"

  raise DependencyFileNotResolvable, msg
end
install_required_python() click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 313
def install_required_python
  # Initialize a git repo to appease pip-tools
  begin
    run_command("git init") if setup_files.any?
  rescue Dependabot::SharedHelpers::HelperSubprocessFailed
    nil
  end

  return if run_command("pyenv versions").include?("#{python_version}\n")

  requirements_path = NativeHelpers.python_requirements_path
  run_command("pyenv install -s #{python_version}")
  run_command("pyenv exec pip install -r "\
              "#{requirements_path}")
end
lockfile() click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 488
def lockfile
  dependency_files.find { |f| f.name == "Pipfile.lock" }
end
normalise(name) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 480
def normalise(name)
  NameNormaliser.normalise(name)
end
pipenv_env_variables() click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 470
def pipenv_env_variables
  {
    "PIPENV_YES" => "true",       # Install new Python ver if needed
    "PIPENV_MAX_RETRIES" => "3",  # Retry timeouts
    "PIPENV_NOSPIN" => "1",       # Don't pollute logs with spinner
    "PIPENV_TIMEOUT" => "600",    # Set install timeout to 10 minutes
    "PIP_DEFAULT_TIMEOUT" => "60" # Set pip timeout to 1 minute
  }
end
pipfile() click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 484
def pipfile
  dependency_files.find { |f| f.name == "Pipfile" }
end
pipfile_content(updated_requirement:) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 342
def pipfile_content(updated_requirement:)
  content = pipfile.content
  content = freeze_other_dependencies(content)
  content = set_target_dependency_req(content, updated_requirement)
  content = add_private_sources(content)
  content
end
python_requirement_parser() click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 440
def python_requirement_parser
  @python_requirement_parser ||=
    FileParser::PythonRequirementParser.new(
      dependency_files: dependency_files
    )
end
python_version() click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 397
def python_version
  @python_version ||= python_version_from_supported_versions
end
python_version_from_supported_versions() click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 401
def python_version_from_supported_versions
  requirement_string =
    if @using_python_two then "2.7.*"
    elsif user_specified_python_requirement
      parts = user_specified_python_requirement.split(".")
      parts.fill("*", (parts.length)..2).join(".")
    else PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.first
    end

  # Ideally, the requirement is satisfied by a Python version we support
  requirement =
    Python::Requirement.requirements_array(requirement_string).first
  version =
    PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.
    find { |v| requirement.satisfied_by?(Python::Version.new(v)) }
  return version if version

  # If not, and changing the patch version would fix things, we do that
  # as the patch version is unlikely to affect resolution
  requirement =
    Python::Requirement.new(requirement_string.gsub(/\.\d+$/, ".*"))
  version =
    PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.
    find { |v| requirement.satisfied_by?(Python::Version.new(v)) }
  return version if version

  # Otherwise we have to raise, giving details of the Python versions
  # that Dependabot supports
  msg = "Dependabot detected the following Python requirement "\
        "for your project: '#{requirement_string}'.\n\nCurrently, the "\
        "following Python versions are supported in Dependabot: "\
        "#{PythonVersions::SUPPORTED_VERSIONS.join(', ')}."
  raise DependencyFileNotResolvable, msg
end
run_command(command, env: {}) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 447
def run_command(command, env: {})
  start = Time.now
  command = SharedHelpers.escape_command(command)
  stdout, process = Open3.capture2e(env, command)
  time_taken = Time.now - start

  return stdout if process.success?

  raise SharedHelpers::HelperSubprocessFailed.new(
    message: stdout,
    error_context: {
      command: command,
      time_taken: time_taken,
      process_exit_value: process.to_s
    }
  )
end
run_pipenv_command(command, env: pipenv_env_variables) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 465
def run_pipenv_command(command, env: pipenv_env_variables)
  run_command("pyenv local #{python_version}")
  run_command(command, env: env)
end
sanitized_setup_file_content(file) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 329
def sanitized_setup_file_content(file)
  @sanitized_setup_file_content ||= {}
  @sanitized_setup_file_content[file.name] ||=
    Python::FileUpdater::SetupFileSanitizer.
    new(setup_file: file, setup_cfg: setup_cfg(file)).
    sanitized_content
end
set_target_dependency_req(pipfile_content, updated_requirement) click to toggle source

rubocop:disable Metrics/PerceivedComplexity

# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 357
def set_target_dependency_req(pipfile_content, updated_requirement)
  return pipfile_content unless updated_requirement

  pipfile_object = TomlRB.parse(pipfile_content)

  %w(packages dev-packages).each do |type|
    names = pipfile_object[type]&.keys || []
    pkg_name = names.find { |nm| normalise(nm) == dependency.name }
    next unless pkg_name || subdep_type?(type)

    pkg_name ||= dependency.name
    if pipfile_object.dig(type, pkg_name).is_a?(Hash)
      pipfile_object[type][pkg_name]["version"] = updated_requirement
    else
      pipfile_object[type][pkg_name] = updated_requirement
    end
  end

  TomlRB.dump(pipfile_object)
end
setup_cfg(file) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 337
def setup_cfg(file)
  config_name = file.name.sub(/\.py$/, ".cfg")
  dependency_files.find { |f| f.name == config_name }
end
setup_cfg_files() click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 496
def setup_cfg_files
  dependency_files.select { |f| f.name.end_with?("setup.cfg") }
end
setup_files() click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 492
def setup_files
  dependency_files.select { |f| f.name.end_with?("setup.py") }
end
subdep_type?(type) click to toggle source

rubocop:enable Metrics/PerceivedComplexity

# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 379
def subdep_type?(type)
  return false if dependency.top_level?

  lockfile_type = Python::FileParser::DEPENDENCY_GROUP_KEYS.
                  find { |i| i.fetch(:pipfile) == type }.
                  fetch(:lockfile)

  JSON.parse(lockfile.content).
    fetch(lockfile_type, {}).
    keys.any? { |k| normalise(k) == dependency.name }
end
user_specified_python_requirement() click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 436
def user_specified_python_requirement
  python_requirement_parser.user_specified_requirements.first
end
write_temporary_dependency_files(updated_req: nil, update_pipfile: true) click to toggle source
# File lib/dependabot/python/update_checker/pipenv_version_resolver.rb, line 282
def write_temporary_dependency_files(updated_req: nil,
                                     update_pipfile: true)
  dependency_files.each do |file|
    path = file.name
    FileUtils.mkdir_p(Pathname.new(path).dirname)
    File.write(path, file.content)
  end

  # Overwrite the .python-version with updated content
  File.write(".python-version", python_version)

  setup_files.each do |file|
    path = file.name
    FileUtils.mkdir_p(Pathname.new(path).dirname)
    File.write(path, sanitized_setup_file_content(file))
  end

  setup_cfg_files.each do |file|
    path = file.name
    FileUtils.mkdir_p(Pathname.new(path).dirname)
    File.write(path, "[metadata]\nname = sanitized-package\n")
  end
  return unless update_pipfile

  # Overwrite the pipfile with updated content
  File.write(
    "Pipfile",
    pipfile_content(updated_requirement: updated_req)
  )
end