class Chef::HTTP

Chef::HTTP

Basic HTTP client, with support for adding features via middleware

Attributes

keepalives[R]
Boolean

if we're doing keepalives or not

middlewares[R]
nethttp_opts[R]

@returns [Hash] options for Net::HTTP to be sent to setters on the object

options[R]
redirect_limit[R]
sign_on_redirect[R]
url[R]

Public Class Methods

middlewares() click to toggle source
# File lib/chef/http.rb, line 66
def self.middlewares
  @middlewares ||= []
end
new(url, options = {}) click to toggle source

Create a HTTP client object. The supplied url is used as the base for all subsequent requests. For example, when initialized with a base url localhost:4000, a call to get with 'nodes' will make an HTTP GET request to localhost:4000/nodes

# File lib/chef/http.rb, line 92
def initialize(url, options = {})
  @url = url
  @default_headers = options[:headers] || {}
  @sign_on_redirect = true
  @redirects_followed = 0
  @redirect_limit = 10
  @keepalives = options[:keepalives] || false
  @options = options
  @nethttp_opts = options[:nethttp] || {}

  @middlewares = []
  self.class.middlewares.each do |middleware_class|
    @middlewares << middleware_class.new(options)
  end
end
use(middleware_class) click to toggle source
# File lib/chef/http.rb, line 70
def self.use(middleware_class)
  middlewares << middleware_class
end

Public Instance Methods

delete(path, headers = {}) click to toggle source

Send an HTTP DELETE request to the path

Parameters

path

path part of the request URL

# File lib/chef/http.rb, line 144
def delete(path, headers = {})
  request(:DELETE, path, headers)
end
get(path, headers = {}) click to toggle source

Send an HTTP GET request to the path

Parameters

path

The path to GET

# File lib/chef/http.rb, line 120
def get(path, headers = {})
  request(:GET, path, headers)
end
head(path, headers = {}) click to toggle source

Send an HTTP HEAD request to the path

Parameters

path

path part of the request URL

# File lib/chef/http.rb, line 112
def head(path, headers = {})
  request(:HEAD, path, headers)
end
http_client(base_url = nil) click to toggle source
# File lib/chef/http.rb, line 263
def http_client(base_url = nil)
  base_url ||= url
  if keepalives && !base_url.nil?
    # only reuse the http_client if we want keepalives and have a base_url
    @http_client ||= {}
    # the per-host per-port cache here gets persistent connections correct when
    # redirecting to different servers
    if base_url.is_a?(String) # sigh, this kind of abuse can't happen with strongly typed languages
      @http_client[base_url] ||= build_http_client(base_url)
    else
      @http_client[base_url.host] ||= {}
      @http_client[base_url.host][base_url.port] ||= build_http_client(base_url)
    end
  else
    build_http_client(base_url)
  end
end
last_response() click to toggle source

DEPRECATED: This is only kept around to provide access to cache control data in lib/chef/provider/remote_file/http.rb FIXME: Find a better API.

# File lib/chef/http.rb, line 284
def last_response
  @last_response
end
post(path, json, headers = {}) click to toggle source

Send an HTTP POST request to the path

Parameters

path

path part of the request URL

# File lib/chef/http.rb, line 136
def post(path, json, headers = {})
  request(:POST, path, headers, json)
end
put(path, json, headers = {}) click to toggle source

Send an HTTP PUT request to the path

Parameters

path

path part of the request URL

# File lib/chef/http.rb, line 128
def put(path, json, headers = {})
  request(:PUT, path, headers, json)
end
request(method, path, headers = {}, data = false) click to toggle source

Makes an HTTP request to path with the given method, headers, and data (if applicable).

# File lib/chef/http.rb, line 150
def request(method, path, headers = {}, data = false)
  http_attempts ||= 0
  url = create_url(path)
  processed_method, url, processed_headers, processed_data = apply_request_middleware(method, url, headers, data)

  response, rest_request, return_value = send_http_request(processed_method, url, processed_headers, processed_data)
  response, rest_request, return_value = apply_response_middleware(response, rest_request, return_value)

  response.error! unless success_response?(response)
  return_value

rescue Net::HTTPClientException => e
  http_attempts += 1
  response = e.response
  if response.is_a?(Net::HTTPNotAcceptable) && version_retries - http_attempts >= 0
    Chef::Log.trace("Negotiating protocol version with #{url}, retry #{http_attempts}/#{version_retries}")
    retry
  else
    raise
  end
rescue Exception => exception
  log_failed_request(response, return_value) unless response.nil?
  raise
end
streaming_request(path, headers = {}, tempfile = nil) { |tempfile| ... } click to toggle source

Makes a streaming download request, streaming the response body to a tempfile. If a block is given, the tempfile is passed to the block and the tempfile will automatically be unlinked after the block is executed.

If no block is given, the tempfile is returned, which means it's up to you to unlink the tempfile when you're done with it.

@yield [tempfile] block to process the tempfile @yieldparam [tempfile<Tempfile>] tempfile

# File lib/chef/http.rb, line 219
def streaming_request(path, headers = {}, tempfile = nil)
  http_attempts ||= 0
  url = create_url(path)
  response, rest_request, return_value = nil, nil, nil
  data = nil

  method = :GET
  method, url, processed_headers, data = apply_request_middleware(method, url, headers, data)

  response, rest_request, return_value = send_http_request(method, url, processed_headers, data) do |http_response|
    if http_response.is_a?(Net::HTTPSuccess)
      tempfile = stream_to_tempfile(url, http_response, tempfile)
    end
    apply_stream_complete_middleware(http_response, rest_request, return_value)
  end

  return nil if response.is_a?(Net::HTTPRedirection)

  unless response.is_a?(Net::HTTPSuccess)
    response.error!
  end

  if block_given?
    begin
      yield tempfile
    ensure
      tempfile && tempfile.close!
    end
  end
  tempfile
rescue Net::HTTPClientException => e
  http_attempts += 1
  response = e.response
  if response.is_a?(Net::HTTPNotAcceptable) && version_retries - http_attempts >= 0
    Chef::Log.trace("Negotiating protocol version with #{url}, retry #{http_attempts}/#{version_retries}")
    retry
  else
    raise
  end
rescue Exception => e
  log_failed_request(response, return_value) unless response.nil?
  raise
end
streaming_request_with_progress(path, headers = {}, tempfile = nil, &progress_block) click to toggle source
# File lib/chef/http.rb, line 175
def streaming_request_with_progress(path, headers = {}, tempfile = nil, &progress_block)
  http_attempts ||= 0
  url = create_url(path)
  response, rest_request, return_value = nil, nil, nil
  data = nil

  method = :GET
  method, url, processed_headers, data = apply_request_middleware(method, url, headers, data)

  response, rest_request, return_value = send_http_request(method, url, processed_headers, data) do |http_response|
    if http_response.is_a?(Net::HTTPSuccess)
      tempfile = stream_to_tempfile(url, http_response, tempfile, &progress_block)
    end
    apply_stream_complete_middleware(http_response, rest_request, return_value)
  end
  return nil if response.is_a?(Net::HTTPRedirection)

  unless response.is_a?(Net::HTTPSuccess)
    response.error!
  end
  tempfile
rescue Net::HTTPClientException => e
  http_attempts += 1
  response = e.response
  if response.is_a?(Net::HTTPNotAcceptable) && version_retries - http_attempts >= 0
    Chef::Log.trace("Negotiating protocol version with #{url}, retry #{http_attempts}/#{version_retries}")
    retry
  else
    raise
  end
rescue Exception => e
  log_failed_request(response, return_value) unless response.nil?
  raise
end

Private Instance Methods

apply_request_middleware(method, url, headers, data) click to toggle source

@api private

# File lib/chef/http.rb, line 339
def apply_request_middleware(method, url, headers, data)
  middlewares.inject([method, url, headers, data]) do |req_data, middleware|
    Chef::Log.trace("Chef::HTTP calling #{middleware.class}#handle_request")
    middleware.handle_request(*req_data)
  end
end
apply_response_middleware(response, rest_request, return_value) click to toggle source

@api private

# File lib/chef/http.rb, line 347
def apply_response_middleware(response, rest_request, return_value)
  middlewares.reverse.inject([response, rest_request, return_value]) do |res_data, middleware|
    Chef::Log.trace("Chef::HTTP calling #{middleware.class}#handle_response")
    middleware.handle_response(*res_data)
  end
end
apply_stream_complete_middleware(response, rest_request, return_value) click to toggle source

@api private

# File lib/chef/http.rb, line 355
def apply_stream_complete_middleware(response, rest_request, return_value)
  middlewares.reverse.inject([response, rest_request, return_value]) do |res_data, middleware|
    Chef::Log.trace("Chef::HTTP calling #{middleware.class}#handle_stream_complete")
    middleware.handle_stream_complete(*res_data)
  end
end
build_headers(method, url, headers = {}, json_body = false) click to toggle source

@api private

# File lib/chef/http.rb, line 521
def build_headers(method, url, headers = {}, json_body = false)
  headers = @default_headers.merge(headers)
  headers["Content-Length"] = json_body.bytesize.to_s if json_body
  headers.merge!(Chef::Config[:custom_http_headers]) if Chef::Config[:custom_http_headers]
  headers
end
build_http_client(base_url) click to toggle source

@api private

# File lib/chef/http.rb, line 306
def build_http_client(base_url)
  if chef_zero_uri?(base_url)
    # PERFORMANCE CRITICAL: *MUST* lazy require here otherwise we load up webrick
    # via chef-zero and that hits DNS (at *require* time) which may timeout,
    # when for most knife/chef-client work we never need/want this loaded.

    unless defined?(SocketlessChefZeroClient)
      require_relative "http/socketless_chef_zero_client"
    end

    SocketlessChefZeroClient.new(base_url)
  else
    BasicClient.new(base_url, ssl_policy: ssl_policy, keepalives: keepalives, nethttp_opts: nethttp_opts)
  end
end
chef_zero_uri?(uri) click to toggle source

@api private

# File lib/chef/http.rb, line 506
def chef_zero_uri?(uri)
  uri = URI.parse(uri) unless uri.respond_to?(:scheme)
  uri.scheme == "chefzero"
end
config() click to toggle source

@api private

# File lib/chef/http.rb, line 489
def config
  Chef::Config
end
create_url(path) click to toggle source

@api private

# File lib/chef/http.rb, line 323
def create_url(path)
  return path if path.is_a?(URI)

  if %r{^(http|https|chefzero)://}i.match?(path)
    URI.parse(path)
  elsif path.nil? || path.empty?
    URI.parse(@url)
  else
    # The regular expressions used here are to make sure '@url' does not have
    # any trailing slashes and 'path' does not have any leading slashes. This
    # way they are always joined correctly using just one slash.
    URI.parse(@url.gsub(%r{/+$}, "") + "/" + path.gsub(%r{^/+}, ""))
  end
end
follow_redirect() { || ... } click to toggle source

@api private

# File lib/chef/http.rb, line 494
def follow_redirect
  raise Chef::Exceptions::RedirectLimitExceeded if @redirects_followed >= redirect_limit

  @redirects_followed += 1
  Chef::Log.trace("Following redirect #{@redirects_followed}/#{redirect_limit}")

  yield
ensure
  @redirects_followed = 0
end
http_disable_auth_on_redirect() click to toggle source

@api private

# File lib/chef/http.rb, line 484
def http_disable_auth_on_redirect
  config[:http_disable_auth_on_redirect]
end
http_retry_count() click to toggle source

@api private

# File lib/chef/http.rb, line 479
def http_retry_count
  options[:http_retry_count] || config[:http_retry_count]
end
http_retry_delay() click to toggle source

@api private

# File lib/chef/http.rb, line 474
def http_retry_delay
  options[:http_retry_delay] || config[:http_retry_delay]
end
log_failed_request(response, return_value) click to toggle source

@api private

# File lib/chef/http.rb, line 363
def log_failed_request(response, return_value)
  return_value ||= {}
  error_message = "HTTP Request Returned #{response.code} #{response.message}: "
  error_message << (return_value["error"].respond_to?(:join) ? return_value["error"].join(", ") : return_value["error"].to_s)
  Chef::Log.info(error_message)
end
redirected_to(response) click to toggle source

@api private

# File lib/chef/http.rb, line 512
def redirected_to(response)
  return nil  unless response.is_a?(Net::HTTPRedirection)
  # Net::HTTPNotModified is undesired subclass of Net::HTTPRedirection so test for this
  return nil  if response.is_a?(Net::HTTPNotModified)

  response["location"]
end
retrying_http_errors(url) { || ... } click to toggle source

Wraps an HTTP request with retry logic.

Arguments

url

URL of the request, used for error messages

@api private

# File lib/chef/http.rb, line 420
def retrying_http_errors(url)
  http_attempts = 0
  begin
    loop do
      http_attempts += 1
      response, request, return_value = yield
      # handle HTTP 50X Error
      if response.is_a?(Net::HTTPServerError) && !Chef::Config.local_mode
        if http_retry_count - http_attempts >= 0
          sleep_time = 1 + (2**http_attempts) + rand(2**http_attempts)
          Chef::Log.warn("Server returned error #{response.code} for #{url}, retrying #{http_attempts}/#{http_retry_count} in #{sleep_time}s") # Updated from error to warn
          sleep(sleep_time)
          redo
        end
      end
      return [response, request, return_value]
    end
  rescue SocketError, Errno::ETIMEDOUT, Errno::ECONNRESET => e
    if http_retry_count - http_attempts >= 0
      Chef::Log.warn("Error connecting to #{url}, retry #{http_attempts}/#{http_retry_count}") # Updated from error to warn
      sleep(http_retry_delay)
      retry
    end
    e.message.replace "Error connecting to #{url} - #{e.message}"
    raise e
  rescue Errno::ECONNREFUSED
    if http_retry_count - http_attempts >= 0
      Chef::Log.warn("Connection refused connecting to #{url}, retry #{http_attempts}/#{http_retry_count}") # Updated from error to warn
      sleep(http_retry_delay)
      retry
    end
    raise Errno::ECONNREFUSED, "Connection refused connecting to #{url}, giving up"
  rescue Timeout::Error
    if http_retry_count - http_attempts >= 0
      Chef::Log.warn("Timeout connecting to #{url}, retry #{http_attempts}/#{http_retry_count}") # Updated from error to warn
      sleep(http_retry_delay)
      retry
    end
    raise Timeout::Error, "Timeout connecting to #{url}, giving up"
  rescue OpenSSL::SSL::SSLError => e
    if (http_retry_count - http_attempts >= 0) && !e.message.include?("certificate verify failed")
      Chef::Log.warn("SSL Error connecting to #{url}, retry #{http_attempts}/#{http_retry_count}") # Updated from error to warn
      sleep(http_retry_delay)
      retry
    end
    raise OpenSSL::SSL::SSLError, "SSL Error connecting to #{url} - #{e.message}"
  end
end
send_http_request(method, url, base_headers, body, &response_handler) click to toggle source

Runs a synchronous HTTP request, with no middleware applied (use request to have the middleware applied). The entire response will be loaded into memory. @api private

# File lib/chef/http.rb, line 378
def send_http_request(method, url, base_headers, body, &response_handler)
  retrying_http_errors(url) do
    headers = build_headers(method, url, base_headers, body)
    client = http_client(url)
    return_value = nil
    if block_given?
      request, response = client.request(method, url, body, headers, &response_handler)
    else
      request, response = client.request(method, url, body, headers, &:read_body)
      return_value = response.read_body
    end
    @last_response = response

    if response.is_a?(Net::HTTPSuccess)
      [response, request, return_value]
    elsif response.is_a?(Net::HTTPNotModified) # Must be tested before Net::HTTPRedirection because it's subclass.
      [response, request, false]
    elsif redirect_location = redirected_to(response)
      if %i{GET HEAD}.include?(method)
        follow_redirect do
          redirected_url = url + redirect_location
          if http_disable_auth_on_redirect
            new_headers = build_headers(method, redirected_url, headers, body)
            new_headers.delete("Authorization") if url.host != redirected_url.host
            send_http_request(method, redirected_url, new_headers, body, &response_handler)
          else
            send_http_request(method, redirected_url, headers, body, &response_handler)
          end
        end
      else
        raise Exceptions::InvalidRedirect, "#{method} request was redirected from #{url} to #{redirect_location}. Only GET and HEAD support redirects."
      end
    else
      [response, request, nil]
    end
  end
end
ssl_policy() click to toggle source

@api private

# File lib/chef/http.rb, line 291
def ssl_policy
  return Chef::HTTP::APISSLPolicy unless @options[:ssl_verify_mode]

  case @options[:ssl_verify_mode]
  when :verify_none
    Chef::HTTP::VerifyNoneSSLPolicy
  when :verify_peer
    Chef::HTTP::VerifyPeerSSLPolicy
  else
    Chef::Log.error("Chef::HTTP was passed an ssl_verify_mode of #{@options[:ssl_verify_mode]} which is unsupported. Falling back to the API policy")
    Chef::HTTP::APISSLPolicy
  end
end
stream_to_tempfile(url, response, tf = nil) { |size, content_length| ... } click to toggle source

@api private

# File lib/chef/http.rb, line 529
def stream_to_tempfile(url, response, tf = nil, &progress_block)
  content_length = response["Content-Length"]
  if tf.nil?
    tf = Tempfile.open("chef-rest")
    if ChefUtils.windows?
      tf.binmode # required for binary files on Windows platforms
    end
  end
  Chef::Log.trace("Streaming download from #{url} to tempfile #{tf.path}")
  # Stolen from http://www.ruby-forum.com/topic/166423
  # Kudos to _why!

  stream_handler = StreamHandler.new(middlewares, response)

  response.read_body do |chunk|
    tf.write(stream_handler.handle_chunk(chunk))
    yield tf.size, content_length if block_given?
  end
  tf.close
  tf
rescue Exception
  tf.close! if tf
  raise
end
success_response?(response) click to toggle source

@api private

# File lib/chef/http.rb, line 371
def success_response?(response)
  response.is_a?(Net::HTTPSuccess) || response.is_a?(Net::HTTPRedirection)
end
version_retries() click to toggle source
# File lib/chef/http.rb, line 469
def version_retries
  @version_retries ||= options[:version_class]&.possible_requests || 1
end