class Dependabot::Python::UpdateChecker::PipCompileVersionResolver
This class does version resolution for pip-compile. Its approach is:
-
Unlock the dependency we're checking in the requirements.in file
-
Run `pip-compile` and see what the result is
rubocop:disable Metrics/ClassLength
Constants
- GIT_DEPENDENCY_UNREACHABLE_REGEX
- GIT_REFERENCE_NOT_FOUND_REGEX
- NATIVE_COMPILATION_ERROR
- VERBOSE_ERROR_OUTPUT_LINES
Attributes
credentials[R]
dependency[R]
dependency_files[R]
Public Class Methods
new(dependency:, dependency_files:, credentials:)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 36 def initialize(dependency:, dependency_files:, credentials:) @dependency = dependency @dependency_files = dependency_files @credentials = credentials @build_isolation = true end
Public Instance Methods
latest_resolvable_version(requirement: nil)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 43 def latest_resolvable_version(requirement: nil) version_string = fetch_latest_resolvable_version_string(requirement: requirement) version_string.nil? ? nil : Python::Version.new(version_string) end
resolvable?(version:)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 50 def resolvable?(version:) @resolvable ||= {} return @resolvable[version] if @resolvable.key?(version) @resolvable[version] = if fetch_latest_resolvable_version_string(requirement: "==#{version}") true else false end end
Private Instance Methods
check_original_requirements_resolvable()
click to toggle source
Needed because pip-compile's resolver isn't perfect. Note: We raise errors from this method, rather than returning a boolean, so that all deps for this repo will raise identical errors when failing to update
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 166 def check_original_requirements_resolvable SharedHelpers.in_a_temporary_directory do SharedHelpers.with_git_configured(credentials: credentials) do write_temporary_dependency_files(update_requirement: false) filenames_to_compile.each do |filename| run_pip_compile_command( "pyenv exec pip-compile #{pip_compile_options(filename)} --allow-unsafe #{filename}" ) end true rescue SharedHelpers::HelperSubprocessFailed => e # Pick the error message that includes resolvability errors, this might be the cause from # handle_pip_compile_errors (it's unclear if we should always pick the cause here) error_message = [e.message, e.cause&.message].compact.find do |msg| ["UnsupportedConstraint", "Could not find a version"].any? { |err| msg.include?(err) } end cleaned_message = clean_error_message(error_message || "") raise if cleaned_message.empty? raise DependencyFileNotResolvable, cleaned_message end end end
clean_error_message(message)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 355 def clean_error_message(message) msg_lines = message.lines msg = msg_lines. take_while { |l| !l.start_with?("During handling of") }. drop_while { |l| l.start_with?(*VERBOSE_ERROR_OUTPUT_LINES) }. join.strip # Redact any URLs, as they may include credentials msg.gsub(/http.*?(?=\s)/, "<redacted>") end
compilation_error?(error)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 108 def compilation_error?(error) error.message.include?(NATIVE_COMPILATION_ERROR) end
compiled_file_for_filename(filename)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 383 def compiled_file_for_filename(filename) compiled_file = compiled_files. find { |f| f.content.match?(output_file_regex(filename)) } compiled_file ||= compiled_files. find { |f| f.name == filename.gsub(/\.in$/, ".txt") } compiled_file end
compiled_file_includes_dependency?(compiled_file)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 399 def compiled_file_includes_dependency?(compiled_file) return false unless compiled_file regex = RequirementParser::INSTALL_REQ_WITH_REQUIREMENT matches = [] compiled_file.content.scan(regex) { matches << Regexp.last_match } matches.any? { |m| normalise(m[:name]) == dependency.name } end
compiled_files()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 516 def compiled_files dependency_files.select { |f| f.name.end_with?(".txt") } end
error_certainly_bad_python_version?(message)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 256 def error_certainly_bad_python_version?(message) return true if message.include?("UnsupportedPythonVersion") unless message.include?('"python setup.py egg_info" failed') || message.include?("exit status 1: python setup.py egg_info") return false end message.include?("SyntaxError") end
fetch_latest_resolvable_version_string(requirement:)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 63 def fetch_latest_resolvable_version_string(requirement:) @latest_resolvable_version_string ||= {} return @latest_resolvable_version_string[requirement] if @latest_resolvable_version_string.key?(requirement) @latest_resolvable_version_string[requirement] ||= SharedHelpers.in_a_temporary_directory do SharedHelpers.with_git_configured(credentials: credentials) do write_temporary_dependency_files(updated_req: requirement) install_required_python filenames_to_compile.each do |filename| # Shell out to pip-compile. # This is slow, as pip-compile needs to do installs. run_pip_compile_command( "pyenv exec pip-compile --allow-unsafe -v "\ "#{pip_compile_options(filename)} -P #{dependency.name} "\ "#{filename}" ) # Run pip-compile a second time, without an update argument, # to ensure it handles markers correctly write_original_manifest_files unless dependency.top_level? run_pip_compile_command( "pyenv exec pip-compile --allow-unsafe "\ "#{pip_compile_options(filename)} #{filename}" ) end # Remove any .python-version file before parsing the reqs FileUtils.remove_entry(".python-version", true) parse_updated_files end rescue SharedHelpers::HelperSubprocessFailed => e retry_count ||= 0 retry_count += 1 if compilation_error?(e) && retry_count <= 1 @build_isolation = false retry end handle_pip_compile_errors(e) end end
filenames_to_compile()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 366 def filenames_to_compile files_from_reqs = dependency.requirements. map { |r| r[:file] }. select { |fn| fn.end_with?(".in") } files_from_compiled_files = pip_compile_files.map(&:name).select do |fn| compiled_file = compiled_file_for_filename(fn) compiled_file_includes_dependency?(compiled_file) end filenames = [*files_from_reqs, *files_from_compiled_files].uniq order_filenames_for_compilation(filenames) end
handle_pip_compile_errors(error)
click to toggle source
rubocop:disable Metrics/AbcSize
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 113 def handle_pip_compile_errors(error) if error.message.include?("Could not find a version") check_original_requirements_resolvable # If the original requirements are resolvable but we get an # incompatibility error after unlocking then it's likely to be # due to problems with pip-compile's cascading resolution return nil end if error.message.include?("UnsupportedConstraint") # If there's an unsupported constraint, check if it existed # previously (and raise if it did) check_original_requirements_resolvable end if (error.message.include?('Command "python setup.py egg_info') || error.message.include?( "exit status 1: python setup.py egg_info" )) && check_original_requirements_resolvable # The latest version of the dependency we're updating is borked # (because it has an unevaluatable setup.py). Skip the update. return end if error.message.include?("Could not find a version ") && !error.message.match?(/#{Regexp.quote(dependency.name)}/i) # Sometimes pip-tools gets confused and can't work around # sub-dependency incompatibilities. Ignore those cases. return nil end if error.message.match?(GIT_REFERENCE_NOT_FOUND_REGEX) name = error.message.match(GIT_REFERENCE_NOT_FOUND_REGEX). named_captures.fetch("name") raise GitDependencyReferenceNotFound, name end if error.message.match?(GIT_DEPENDENCY_UNREACHABLE_REGEX) url = error.message.match(GIT_DEPENDENCY_UNREACHABLE_REGEX). named_captures.fetch("url") raise GitDependenciesNotReachable, url end raise end
install_required_python()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 302 def install_required_python return if run_command("pyenv versions").include?("#{python_version}\n") run_command("pyenv install -s #{python_version}") run_command("pyenv exec pip install -r"\ "#{NativeHelpers.python_requirements_path}") end
normalise(name)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 341 def normalise(name) NameNormaliser.normalise(name) end
order_filenames_for_compilation(filenames)
click to toggle source
If the files we need to update require one another then we need to update them in the right order
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 411 def order_filenames_for_compilation(filenames) ordered_filenames = [] while (remaining_filenames = filenames - ordered_filenames).any? ordered_filenames += remaining_filenames. select do |fn| unupdated_reqs = requirement_map[fn] - ordered_filenames (unupdated_reqs & filenames).empty? end end ordered_filenames end
output_file_regex(filename)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 395 def output_file_regex(filename) "--output-file[=\s]+.*\s#{Regexp.escape(filename)}\s*$" end
parse_updated_files()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 445 def parse_updated_files updated_files = dependency_files.map do |file| next file if file.name == ".python-version" updated_file = file.dup updated_file.content = File.read(file.name) updated_file end Python::FileParser.new( dependency_files: updated_files, source: nil, credentials: credentials ).parse.find { |d| d.name == dependency.name }&.version end
pip_compile_files()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 512 def pip_compile_files dependency_files.select { |f| f.name.end_with?(".in") } end
pip_compile_index_options()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 222 def pip_compile_index_options credentials. select { |cred| cred["type"] == "python_index" }. map do |cred| authed_url = AuthedUrlBuilder.authed_url(credential: cred) if cred["replaces-base"] "--index-url=#{authed_url}" else "--extra-index-url=#{authed_url}" end end end
pip_compile_options(filename)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 211 def pip_compile_options(filename) options = @build_isolation ? ["--build-isolation"] : ["--no-build-isolation"] options += pip_compile_index_options if (requirements_file = compiled_file_for_filename(filename)) options << "--output-file=#{requirements_file.name}" end options.join(" ") end
pre_installed_python?(version)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 504 def pre_installed_python?(version) PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.include?(version) end
python_env()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 241 def python_env env = {} # Handle Apache Airflow 1.10.x installs if dependency_files.any? { |f| f.content.include?("apache-airflow") } if dependency_files.any? { |f| f.content.include?("unidecode") } env["AIRFLOW_GPL_UNIDECODE"] = "yes" else env["SLUGIFY_USES_TEXT_UNIDECODE"] = "yes" end end env end
python_requirement_parser()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 497 def python_requirement_parser @python_requirement_parser ||= FileParser::PythonRequirementParser.new( dependency_files: dependency_files ) end
python_version()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 462 def python_version @python_version ||= user_specified_python_version || python_version_matching_imputed_requirements || PythonVersions::PRE_INSTALLED_PYTHON_VERSIONS.first end
python_version_matching(requirements)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 486 def python_version_matching(requirements) PythonVersions::SUPPORTED_VERSIONS_TO_ITERATE.find do |version_string| version = Python::Version.new(version_string) requirements.all? do |req| next req.any? { |r| r.satisfied_by?(version) } if req.is_a?(Array) req.satisfied_by?(version) end end end
python_version_matching_imputed_requirements()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 478 def python_version_matching_imputed_requirements compiled_file_python_requirement_markers = python_requirement_parser.imputed_requirements.map do |r| Dependabot::Python::Requirement.new(r) end python_version_matching(compiled_file_python_requirement_markers) end
requirement_map()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 426 def requirement_map child_req_regex = Python::FileFetcher::CHILD_REQUIREMENT_REGEX @requirement_map ||= pip_compile_files.each_with_object({}) do |file, req_map| paths = file.content.scan(child_req_regex).flatten current_dir = File.dirname(file.name) req_map[file.name] = paths.map do |path| path = File.join(current_dir, path) if current_dir != "." path = Pathname.new(path).cleanpath.to_path path = path.gsub(/\.txt$/, ".in") next if path == file.name path end.uniq.compact end end
run_command(command, env: python_env)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 193 def run_command(command, env: python_env) start = Time.now command = SharedHelpers.escape_command(command) stdout, process = Open3.capture2e(env, command) time_taken = Time.now - start return stdout if process.success? raise SharedHelpers::HelperSubprocessFailed.new( message: stdout, error_context: { command: command, time_taken: time_taken, process_exit_value: process.to_s } ) end
run_pip_compile_command(command)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 236 def run_pip_compile_command(command) run_command("pyenv local #{python_version}") run_command(command) end
sanitized_setup_file_content(file)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 310 def sanitized_setup_file_content(file) @sanitized_setup_file_content ||= {} return @sanitized_setup_file_content[file.name] if @sanitized_setup_file_content[file.name] @sanitized_setup_file_content[file.name] = Python::FileUpdater::SetupFileSanitizer. new(setup_file: file, setup_cfg: setup_cfg(file)). sanitized_content end
setup_cfg(file)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 320 def setup_cfg(file) dependency_files.find do |f| f.name == file.name.sub(/\.py$/, ".cfg") end end
setup_cfg_files()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 520 def setup_cfg_files dependency_files.select { |f| f.name.end_with?("setup.cfg") } end
setup_files()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 508 def setup_files dependency_files.select { |f| f.name.end_with?("setup.py") } end
update_req_file(file, updated_req)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 326 def update_req_file(file, updated_req) return file.content unless file.name.end_with?(".in") req = dependency.requirements.find { |r| r[:file] == file.name } return file.content + "\n#{dependency.name} #{updated_req}" unless req&.fetch(:requirement) Python::FileUpdater::RequirementReplacer.new( content: file.content, dependency_name: dependency.name, old_requirement: req[:requirement], new_requirement: updated_req ).updated_content end
user_specified_python_version()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 469 def user_specified_python_version return unless python_requirement_parser.user_specified_requirements.any? user_specified_requirements = python_requirement_parser.user_specified_requirements. map { |r| Python::Requirement.requirements_array(r) } python_version_matching(user_specified_requirements) end
write_original_manifest_files()
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 295 def write_original_manifest_files pip_compile_files.each do |file| FileUtils.mkdir_p(Pathname.new(file.name).dirname) File.write(file.name, file.content) end end
write_temporary_dependency_files(updated_req: nil, update_requirement: true)
click to toggle source
# File lib/dependabot/python/update_checker/pip_compile_version_resolver.rb, line 267 def write_temporary_dependency_files(updated_req: nil, update_requirement: true) dependency_files.each do |file| path = file.name FileUtils.mkdir_p(Pathname.new(path).dirname) updated_content = if update_requirement then update_req_file(file, updated_req) else file.content end File.write(path, updated_content) end # Overwrite the .python-version with updated content File.write(".python-version", python_version) setup_files.each do |file| path = file.name FileUtils.mkdir_p(Pathname.new(path).dirname) File.write(path, sanitized_setup_file_content(file)) end setup_cfg_files.each do |file| path = file.name FileUtils.mkdir_p(Pathname.new(path).dirname) File.write(path, "[metadata]\nname = sanitized-package\n") end end