class Chef::HTTP

Chef::HTTP

Basic HTTP client, with support for adding features via middleware

Attributes

keepalives[R]
Boolean

if we're doing keepalives or not

middlewares[R]
options[R]
redirect_limit[R]
sign_on_redirect[R]
url[R]

Public Class Methods

middlewares() click to toggle source
# File lib/chef/http.rb, line 64
def self.middlewares
  @middlewares ||= []
end
new(url, options = {}) click to toggle source

Create a HTTP client object. The supplied url is used as the base for all subsequent requests. For example, when initialized with a base url localhost:4000, a call to get with 'nodes' will make an HTTP GET request to localhost:4000/nodes

# File lib/chef/http.rb, line 87
def initialize(url, options = {})
  @url = url
  @default_headers = options[:headers] || {}
  @sign_on_redirect = true
  @redirects_followed = 0
  @redirect_limit = 10
  @keepalives = options[:keepalives] || false
  @options = options

  @middlewares = []
  self.class.middlewares.each do |middleware_class|
    @middlewares << middleware_class.new(options)
  end
end
use(middleware_class) click to toggle source
# File lib/chef/http.rb, line 68
def self.use(middleware_class)
  middlewares << middleware_class
end

Public Instance Methods

delete(path, headers = {}) click to toggle source

Send an HTTP DELETE request to the path

Parameters

path

path part of the request URL

# File lib/chef/http.rb, line 138
def delete(path, headers = {})
  request(:DELETE, path, headers)
end
get(path, headers = {}) click to toggle source

Send an HTTP GET request to the path

Parameters

path

The path to GET

# File lib/chef/http.rb, line 114
def get(path, headers = {})
  request(:GET, path, headers)
end
head(path, headers = {}) click to toggle source

Send an HTTP HEAD request to the path

Parameters

path

path part of the request URL

# File lib/chef/http.rb, line 106
def head(path, headers = {})
  request(:HEAD, path, headers)
end
http_client(base_url = nil) click to toggle source
# File lib/chef/http.rb, line 265
def http_client(base_url = nil)
  base_url ||= url
  if keepalives && !base_url.nil?
    # only reuse the http_client if we want keepalives and have a base_url
    @http_client ||= {}
    # the per-host per-port cache here gets peristent connections correct when
    # redirecting to different servers
    if base_url.is_a?(String) # sigh, this kind of abuse can't happen with strongly typed languages
      @http_client[base_url] ||= build_http_client(base_url)
    else
      @http_client[base_url.host] ||= {}
      @http_client[base_url.host][base_url.port] ||= build_http_client(base_url)
    end
  else
    build_http_client(base_url)
  end
end
last_response() click to toggle source

DEPRECATED: This is only kept around to provide access to cache control data in lib/chef/provider/remote_file/http.rb FIXME: Find a better API.

# File lib/chef/http.rb, line 286
def last_response
  @last_response
end
post(path, json, headers = {}) click to toggle source

Send an HTTP POST request to the path

Parameters

path

path part of the request URL

# File lib/chef/http.rb, line 130
def post(path, json, headers = {})
  request(:POST, path, headers, json)
end
put(path, json, headers = {}) click to toggle source

Send an HTTP PUT request to the path

Parameters

path

path part of the request URL

# File lib/chef/http.rb, line 122
def put(path, json, headers = {})
  request(:PUT, path, headers, json)
end
request(method, path, headers = {}, data = false) click to toggle source

Makes an HTTP request to path with the given method, headers, and data (if applicable).

# File lib/chef/http.rb, line 144
def request(method, path, headers = {}, data = false)
  http_attempts ||= 0
  url = create_url(path)
  processed_method, url, processed_headers, processed_data = apply_request_middleware(method, url, headers, data)

  response, rest_request, return_value = send_http_request(processed_method, url, processed_headers, processed_data)
  response, rest_request, return_value = apply_response_middleware(response, rest_request, return_value)

  response.error! unless success_response?(response)
  return_value

rescue Net::HTTPServerException => e
  http_attempts += 1
  response = e.response
  if response.kind_of?(Net::HTTPNotAcceptable) && version_retries - http_attempts > 0
    Chef::Log.trace("Negotiating protocol version with #{url}, retry #{http_attempts}/#{version_retries}")
    retry
  else
    raise
  end
rescue Exception => exception
  log_failed_request(response, return_value) unless response.nil?

  if exception.respond_to?(:chef_rest_request=)
    exception.chef_rest_request = rest_request
  end
  raise
end
streaming_request(path, headers = {}, tempfile = nil) { |tempfile| ... } click to toggle source

Makes a streaming download request, streaming the response body to a tempfile. If a block is given, the tempfile is passed to the block and the tempfile will automatically be unlinked after the block is executed.

If no block is given, the tempfile is returned, which means it's up to you to unlink the tempfile when you're done with it.

@yield [tempfile] block to process the tempfile @yieldparams [tempfile<Tempfile>] tempfile

# File lib/chef/http.rb, line 219
def streaming_request(path, headers = {}, tempfile = nil)
  http_attempts ||= 0
  url = create_url(path)
  response, rest_request, return_value = nil, nil, nil
  data = nil

  method = :GET
  method, url, processed_headers, data = apply_request_middleware(method, url, headers, data)

  response, rest_request, return_value = send_http_request(method, url, processed_headers, data) do |http_response|
    if http_response.kind_of?(Net::HTTPSuccess)
      tempfile = stream_to_tempfile(url, http_response, tempfile)
    end
    apply_stream_complete_middleware(http_response, rest_request, return_value)
  end

  return nil if response.kind_of?(Net::HTTPRedirection)
  unless response.kind_of?(Net::HTTPSuccess)
    response.error!
  end

  if block_given?
    begin
      yield tempfile
    ensure
      tempfile && tempfile.close!
    end
  end
  tempfile
rescue Net::HTTPServerException => e
  http_attempts += 1
  response = e.response
  if response.kind_of?(Net::HTTPNotAcceptable) && version_retries - http_attempts > 0
    Chef::Log.trace("Negotiating protocol version with #{url}, retry #{http_attempts}/#{version_retries}")
    retry
  else
    raise
  end
rescue Exception => e
  log_failed_request(response, return_value) unless response.nil?
  if e.respond_to?(:chef_rest_request=)
    e.chef_rest_request = rest_request
  end
  raise
end
streaming_request_with_progress(path, headers = {}, tempfile = nil, &progress_block) click to toggle source
# File lib/chef/http.rb, line 173
def streaming_request_with_progress(path, headers = {}, tempfile = nil, &progress_block)
  http_attempts ||= 0
  url = create_url(path)
  response, rest_request, return_value = nil, nil, nil
  data = nil

  method = :GET
  method, url, processed_headers, data = apply_request_middleware(method, url, headers, data)

  response, rest_request, return_value = send_http_request(method, url, processed_headers, data) do |http_response|
    if http_response.kind_of?(Net::HTTPSuccess)
      tempfile = stream_to_tempfile(url, http_response, tempfile, &progress_block)
    end
    apply_stream_complete_middleware(http_response, rest_request, return_value)
  end
  return nil if response.kind_of?(Net::HTTPRedirection)
  unless response.kind_of?(Net::HTTPSuccess)
    response.error!
  end
  tempfile
rescue Net::HTTPServerException => e
  http_attempts += 1
  response = e.response
  if response.kind_of?(Net::HTTPNotAcceptable) && version_retries - http_attempts > 0
    Chef::Log.trace("Negotiating protocol version with #{url}, retry #{http_attempts}/#{version_retries}")
    retry
  else
    raise
  end
rescue Exception => e
  log_failed_request(response, return_value) unless response.nil?
  if e.respond_to?(:chef_rest_request=)
    e.chef_rest_request = rest_request
  end
  raise
end

Private Instance Methods

apply_request_middleware(method, url, headers, data) click to toggle source

@api private

# File lib/chef/http.rb, line 325
def apply_request_middleware(method, url, headers, data)
  middlewares.inject([method, url, headers, data]) do |req_data, middleware|
    Chef::Log.trace("Chef::HTTP calling #{middleware.class}#handle_request")
    middleware.handle_request(*req_data)
  end
end
apply_response_middleware(response, rest_request, return_value) click to toggle source

@api private

# File lib/chef/http.rb, line 333
def apply_response_middleware(response, rest_request, return_value)
  middlewares.reverse.inject([response, rest_request, return_value]) do |res_data, middleware|
    Chef::Log.trace("Chef::HTTP calling #{middleware.class}#handle_response")
    middleware.handle_response(*res_data)
  end
end
apply_stream_complete_middleware(response, rest_request, return_value) click to toggle source

@api private

# File lib/chef/http.rb, line 341
def apply_stream_complete_middleware(response, rest_request, return_value)
  middlewares.reverse.inject([response, rest_request, return_value]) do |res_data, middleware|
    Chef::Log.trace("Chef::HTTP calling #{middleware.class}#handle_stream_complete")
    middleware.handle_stream_complete(*res_data)
  end
end
build_headers(method, url, headers = {}, json_body = false) click to toggle source

@api private

# File lib/chef/http.rb, line 505
def build_headers(method, url, headers = {}, json_body = false)
  headers = @default_headers.merge(headers)
  headers["Content-Length"] = json_body.bytesize.to_s if json_body
  headers.merge!(Chef::Config[:custom_http_headers]) if Chef::Config[:custom_http_headers]
  headers
end
build_http_client(base_url) click to toggle source

@api private

# File lib/chef/http.rb, line 293
def build_http_client(base_url)
  if chef_zero_uri?(base_url)
    # PERFORMANCE CRITICAL: *MUST* lazy require here otherwise we load up webrick
    # via chef-zero and that hits DNS (at *require* time) which may timeout,
    # when for most knife/chef-client work we never need/want this loaded.

    unless defined?(SocketlessChefZeroClient)
      require "chef/http/socketless_chef_zero_client"
    end

    SocketlessChefZeroClient.new(base_url)
  else
    BasicClient.new(base_url, ssl_policy: Chef::HTTP::APISSLPolicy, keepalives: keepalives)
  end
end
chef_zero_uri?(uri) click to toggle source

@api private

# File lib/chef/http.rb, line 491
def chef_zero_uri?(uri)
  uri = URI.parse(uri) unless uri.respond_to?(:scheme)
  uri.scheme == "chefzero"
end
config() click to toggle source

@api private

# File lib/chef/http.rb, line 475
def config
  Chef::Config
end
create_url(path) click to toggle source

@api private

# File lib/chef/http.rb, line 310
def create_url(path)
  return path if path.is_a?(URI)
  if path =~ /^(http|https|chefzero):\/\//i
    URI.parse(path)
  elsif path.nil? || path.empty?
    URI.parse(@url)
  else
    # The regular expressions used here are to make sure '@url' does not have
    # any trailing slashes and 'path' does not have any leading slashes. This
    # way they are always joined correctly using just one slash.
    URI.parse(@url.gsub(%r{/+$}, "") + "/" + path.gsub(%r{^/+}, ""))
  end
end
follow_redirect() { || ... } click to toggle source

@api private

# File lib/chef/http.rb, line 480
def follow_redirect
  raise Chef::Exceptions::RedirectLimitExceeded if @redirects_followed >= redirect_limit
  @redirects_followed += 1
  Chef::Log.trace("Following redirect #{@redirects_followed}/#{redirect_limit}")

  yield
ensure
  @redirects_followed = 0
end
http_disable_auth_on_redirect() click to toggle source

@api private

# File lib/chef/http.rb, line 470
def http_disable_auth_on_redirect
  config[:http_disable_auth_on_redirect]
end
http_retry_count() click to toggle source

@api private

# File lib/chef/http.rb, line 465
def http_retry_count
  config[:http_retry_count]
end
http_retry_delay() click to toggle source

@api private

# File lib/chef/http.rb, line 460
def http_retry_delay
  config[:http_retry_delay]
end
log_failed_request(response, return_value) click to toggle source

@api private

# File lib/chef/http.rb, line 349
def log_failed_request(response, return_value)
  return_value ||= {}
  error_message = "HTTP Request Returned #{response.code} #{response.message}: "
  error_message << (return_value["error"].respond_to?(:join) ? return_value["error"].join(", ") : return_value["error"].to_s)
  Chef::Log.info(error_message)
end
redirected_to(response) click to toggle source

@api private

# File lib/chef/http.rb, line 497
def redirected_to(response)
  return nil  unless response.kind_of?(Net::HTTPRedirection)
  # Net::HTTPNotModified is undesired subclass of Net::HTTPRedirection so test for this
  return nil  if response.kind_of?(Net::HTTPNotModified)
  response["location"]
end
retrying_http_errors(url) { || ... } click to toggle source

Wraps an HTTP request with retry logic.

Arguments

url

URL of the request, used for error messages

@api private

# File lib/chef/http.rb, line 406
def retrying_http_errors(url)
  http_attempts = 0
  begin
    loop do
      http_attempts += 1
      response, request, return_value = yield
      # handle HTTP 50X Error
      if response.kind_of?(Net::HTTPServerError) && !Chef::Config.local_mode
        if http_retry_count - http_attempts + 1 > 0
          sleep_time = 1 + (2**http_attempts) + rand(2**http_attempts)
          Chef::Log.error("Server returned error #{response.code} for #{url}, retrying #{http_attempts}/#{http_retry_count} in #{sleep_time}s")
          sleep(sleep_time)
          redo
        end
      end
      return [response, request, return_value]
    end
  rescue SocketError, Errno::ETIMEDOUT, Errno::ECONNRESET => e
    if http_retry_count - http_attempts + 1 > 0
      Chef::Log.error("Error connecting to #{url}, retry #{http_attempts}/#{http_retry_count}")
      sleep(http_retry_delay)
      retry
    end
    e.message.replace "Error connecting to #{url} - #{e.message}"
    raise e
  rescue Errno::ECONNREFUSED
    if http_retry_count - http_attempts + 1 > 0
      Chef::Log.error("Connection refused connecting to #{url}, retry #{http_attempts}/#{http_retry_count}")
      sleep(http_retry_delay)
      retry
    end
    raise Errno::ECONNREFUSED, "Connection refused connecting to #{url}, giving up"
  rescue Timeout::Error
    if http_retry_count - http_attempts + 1 > 0
      Chef::Log.error("Timeout connecting to #{url}, retry #{http_attempts}/#{http_retry_count}")
      sleep(http_retry_delay)
      retry
    end
    raise Timeout::Error, "Timeout connecting to #{url}, giving up"
  rescue OpenSSL::SSL::SSLError => e
    if (http_retry_count - http_attempts + 1 > 0) && !e.message.include?("certificate verify failed")
      Chef::Log.error("SSL Error connecting to #{url}, retry #{http_attempts}/#{http_retry_count}")
      sleep(http_retry_delay)
      retry
    end
    raise OpenSSL::SSL::SSLError, "SSL Error connecting to #{url} - #{e.message}"
  end
end
send_http_request(method, url, base_headers, body, &response_handler) click to toggle source

Runs a synchronous HTTP request, with no middleware applied (use request to have the middleware applied). The entire response will be loaded into memory. @api private

# File lib/chef/http.rb, line 364
def send_http_request(method, url, base_headers, body, &response_handler)
  retrying_http_errors(url) do
    headers = build_headers(method, url, base_headers, body)
    client = http_client(url)
    return_value = nil
    if block_given?
      request, response = client.request(method, url, body, headers, &response_handler)
    else
      request, response = client.request(method, url, body, headers) { |r| r.read_body }
      return_value = response.read_body
    end
    @last_response = response

    if response.kind_of?(Net::HTTPSuccess)
      [response, request, return_value]
    elsif response.kind_of?(Net::HTTPNotModified) # Must be tested before Net::HTTPRedirection because it's subclass.
      [response, request, false]
    elsif redirect_location = redirected_to(response)
      if [:GET, :HEAD].include?(method)
        follow_redirect do
          redirected_url = url + redirect_location
          if http_disable_auth_on_redirect
            new_headers = build_headers(method, redirected_url, headers, body)
            new_headers.delete("Authorization") if url.host != redirected_url.host
            send_http_request(method, redirected_url, new_headers, body, &response_handler)
          else
            send_http_request(method, redirected_url, headers, body, &response_handler)
          end
        end
      else
        raise Exceptions::InvalidRedirect, "#{method} request was redirected from #{url} to #{redirect_location}. Only GET and HEAD support redirects."
      end
    else
      [response, request, nil]
    end
  end
end
stream_to_tempfile(url, response, tf = nil) { |size, content_length| ... } click to toggle source

@api private

# File lib/chef/http.rb, line 513
def stream_to_tempfile(url, response, tf = nil, &progress_block)
  content_length = response["Content-Length"]
  if tf.nil?
    tf = Tempfile.open("chef-rest")
    if Chef::Platform.windows?
      tf.binmode # required for binary files on Windows platforms
    end
  end
  Chef::Log.trace("Streaming download from #{url} to tempfile #{tf.path}")
  # Stolen from http://www.ruby-forum.com/topic/166423
  # Kudos to _why!

  stream_handler = StreamHandler.new(middlewares, response)

  response.read_body do |chunk|
    tf.write(stream_handler.handle_chunk(chunk))
    yield tf.size, content_length if block_given?
  end
  tf.close
  tf
rescue Exception
  tf.close! if tf
  raise
end
success_response?(response) click to toggle source

@api private

# File lib/chef/http.rb, line 357
def success_response?(response)
  response.kind_of?(Net::HTTPSuccess) || response.kind_of?(Net::HTTPRedirection)
end
version_retries() click to toggle source
# File lib/chef/http.rb, line 455
def version_retries
  @version_retries ||= options[:version_class].possible_requests
end