class TFWrapper::RakeTasks

Generates Rake tasks for working with Terraform.

Attributes

instance[RW]

set when installed

tf_version[R]

Public Class Methods

install_tasks(tf_dir, opts = {}) click to toggle source

Install the Rake tasks for working with Terraform. For full parameter documentation, see {TFWrapper::RakeTasks#initialize}

@param (see initialize)

# File lib/tfwrapper/raketasks.rb, line 31
def install_tasks(tf_dir, opts = {})
  new(tf_dir, opts).install
end
new(tf_dir, opts = {}) click to toggle source

Generate Rake tasks for working with Terraform.

@param tf_dir [String] Terraform config directory, relative to Rakefile.

Set to '.' if the Rakefile is in the same directory as the ``.tf``
configuration files.

@option opts [Hash] :backend_config hash of Terraform remote state

backend configuration options, to override or supplement those in
the terraform configuration. See the
[Remote State](https://www.terraform.io/docs/state/remote.html)
documentation for further information.

@option opts [String] :namespace_prefix if specified and not nil, this

will put all tasks in a "#{namespace_prefix}_tf:" namespace instead
of "tf:". This allows using manheim_helpers for multiple terraform
configurations in the same Rakefile.

@option opts [Hash] :tf_vars_from_env hash of Terraform variables to the

(required) environment variables to populate their values from

@option opts [Hash] :allowed_empty_vars array of environment variable

names (specified in :tf_vars_from_env) to allow to be empty or missing.

@option opts [Hash] :tf_extra_vars hash of Terraform variables to their

values; overrides any same-named keys in ``tf_vars_from_env``

@option opts [Array] :tf_sensitive_vars list of Terraform variables

which should not be printed

@option opts [String] :consul_url URL to access Consul at, for the

``:consul_env_vars_prefix`` option.

@option opts [String] :consul_env_vars_prefix if specified and not nil,

write the environment variables used from ``tf_vars_from_env``
and their values to JSON at this path in Consul. This should have
the same naming constraints as ``consul_prefix``.

@option opts [Proc] :before_proc Proc instance to call before executing

the body of each task. Called with two arguments, the String full
(namespaced) name of the task being executed, and ``tf_dir``. Returning
or breaking from this Proc will cause the task to not execute; to exit
the Proc early, use ``next``.

@option opts [Proc] :after_proc Proc instance to call after executing

the body of each task. Called with two arguments, the String full
(namespaced) name of the task being executed, and ``tf_dir``. This will
not execute if the body of the task fails.

@option opts [Bool] :disable_landscape By default, if the

``terraform_landscape`` gem can be loaded, it will be used to reformat
the output of ``terraform plan``. If this is not desired, set to
``true`` to disable landscale. Default: ``false``.

@option opts [Symbol, nil] :landscape_progress The “terraform_landscape“

code used to reformat plan output requires the full output of the
complete ``plan`` execution. By default, this means that when landscape
is used, no output will appear from the time ``terraform plan`` begins
until the command is complete. If progress output is desired, this option
can be set to one of the following: ``:dots`` to print a dot to STDOUT
for every line of ``terraform plan`` output, ``:lines`` to print a dot
followed by a newline (e.g. for systems like Jenkins that line buffer)
for every line of ``terraform plan`` output, or ``:stream`` to stream
the raw ``terraform plan`` output (which will then be followed by the
reformatted landscape output). Default is ``nil`` to show no progress
output.
# File lib/tfwrapper/raketasks.rb, line 95
def initialize(tf_dir, opts = {})
  # find the directory that contains the Rakefile
  rakedir = File.realpath(Rake.application.rakefile)
  rakedir = File.dirname(rakedir) if File.file?(rakedir)
  @tf_dir = File.realpath(File.join(rakedir, tf_dir))
  @ns_prefix = opts.fetch(:namespace_prefix, nil)
  @consul_env_vars_prefix = opts.fetch(:consul_env_vars_prefix, nil)
  @tf_vars_from_env = opts.fetch(:tf_vars_from_env, {})
  @allowed_empty_vars = opts.fetch(:allowed_empty_vars, [])
  @tf_sensitive_vars = opts.fetch(:tf_sensitive_vars, [])
  @tf_extra_vars = opts.fetch(:tf_extra_vars, {})
  @backend_config = opts.fetch(:backend_config, {})
  @consul_url = opts.fetch(:consul_url, nil)
  @disable_landscape = opts.fetch(:disable_landscape, false)
  @landscape_progress = opts.fetch(:landscape_progress, nil)
  unless [:dots, :lines, :stream, nil].include?(@landscape_progress)
    raise(
      ArgumentError,
      'landscape_progress option must be one of: ' \
      '[:dots, :lines, :stream, nil]'
    )
  end
  @before_proc = opts.fetch(:before_proc, nil)
  if !@before_proc.nil? && !@before_proc.is_a?(Proc)
    raise(
      TypeError,
      'TFWrapper::RakeTasks.initialize option :before_proc must be a ' \
      'Proc instance, not a ' + @before_proc.class.name
    )
  end
  @after_proc = opts.fetch(:after_proc, nil)
  if !@after_proc.nil? && !@after_proc.is_a?(Proc)
    raise(
      TypeError,
      'TFWrapper::RakeTasks.initialize option :after_proc must be a Proc ' \
      'instance, not a ' + @after_proc.class.name
    )
  end
  # default to lowest possible version; this is set in the 'init' task
  @tf_version = Gem::Version.new('0.0.0')
  # rubocop:disable Style/GuardClause
  if @consul_url.nil? && !@consul_env_vars_prefix.nil?
    raise StandardError, 'Cannot set env vars in Consul when consul_url ' \
      'option is nil.'
  end
  # rubocop:enable Style/GuardClause
end

Public Instance Methods

check_tf_version() click to toggle source

Check that the terraform version is compatible

# File lib/tfwrapper/raketasks.rb, line 428
def check_tf_version
  # run: terraform -version
  all_out_err, exit_status = TFWrapper::Helpers.run_cmd_stream_output(
    'terraform version', @tf_dir
  )
  unless exit_status.zero?
    raise StandardError, "ERROR: 'terraform -version' exited " \
      "#{exit_status}: #{all_out_err}"
  end
  all_out_err = all_out_err.strip
  # Find the terraform version string
  m = /Terraform v(\d+\.\d+\.\d+).*/.match(all_out_err)
  unless m
    raise StandardError, 'ERROR: could not determine terraform version ' \
      "from 'terraform -version' output: #{all_out_err}"
  end
  # the version will be a string like:
  # Terraform v0.9.2
  # or:
  # Terraform v0.9.3-dev (<GIT SHA><+CHANGES>)
  tf_ver = Gem::Version.new(m[1])
  unless tf_ver >= min_tf_version
    raise StandardError, "ERROR: tfwrapper #{TFWrapper::VERSION} is only " \
      "compatible with Terraform >= #{min_tf_version} but your terraform " \
      "binary reports itself as #{m[1]} (#{all_out_err})"
  end
  puts "Running with: #{all_out_err}"
  tf_ver
end
cmd_with_targets(cmd_array, target, extras) click to toggle source

Create a terraform command line with optional targets specified; targets are inserted between cmd_array and suffix_array.

This is intended to simplify parsing Rake task arguments and inserting them into the command as targets; to get a Rake task to take a variable number of arguments, we define a first argument (“:target“) which is either a String or nil. Any additional arguments specified end up in “args.extras“, which is either nil or an Array of additional String arguments.

@param cmd_array [Array] array of the beginning parts of the terraform

command; usually something like:
['terraform', 'ACTION', '-var'file', 'VAR_FILE_PATH']

@param target [String] the first target parameter given to the Rake

task, or nil.

@param extras [Array] array of additional target parameters given to the

Rake task, or nil.
# File lib/tfwrapper/raketasks.rb, line 502
def cmd_with_targets(cmd_array, target, extras)
  final_arr = cmd_array
  final_arr.concat(['-target', target]) unless target.nil?
  # rubocop:disable Style/SafeNavigation
  extras.each { |e| final_arr.concat(['-target', e]) } unless extras.nil?
  # rubocop:enable Style/SafeNavigation
  final_arr.join(' ')
end
install() click to toggle source

install all Rake tasks - calls other install_* methods rubocop:disable Metrics/CyclomaticComplexity

# File lib/tfwrapper/raketasks.rb, line 153
def install
  install_init
  install_plan
  install_apply
  install_refresh
  install_destroy
  install_write_tf_vars
  install_output
end
install_apply() click to toggle source

add the 'tf:apply' Rake task

# File lib/tfwrapper/raketasks.rb, line 218
def install_apply
  namespace nsprefix do
    desc 'Apply a terraform plan that will provision your resources; ' \
      'specify optional CSV targets'
    task :apply, [:target] => [
      :"#{nsprefix}:init",
      :"#{nsprefix}:write_tf_vars",
      :"#{nsprefix}:plan"
    ] do |t, args|
      @before_proc.call(t.name, @tf_dir) unless @before_proc.nil?
      cmd_arr = %w[terraform apply]
      cmd_arr << '-auto-approve' if tf_version >= Gem::Version.new('0.10.0')
      cmd_arr << "-var-file #{var_file_path}"
      cmd = cmd_with_targets(
        cmd_arr,
        args[:target],
        args.extras
      )
      terraform_runner(cmd)

      update_consul_stack_env_vars unless @consul_env_vars_prefix.nil?
      @after_proc.call(t.name, @tf_dir) unless @after_proc.nil?
    end
  end
end
install_destroy() click to toggle source

add the 'tf:destroy' Rake task

# File lib/tfwrapper/raketasks.rb, line 287
def install_destroy
  namespace nsprefix do
    desc 'Destroy any live resources that are tracked by your state ' \
      'files; specify optional CSV targets'
    task :destroy, [:target] => [
      :"#{nsprefix}:init",
      :"#{nsprefix}:write_tf_vars"
    ] do |t, args|
      @before_proc.call(t.name, @tf_dir) unless @before_proc.nil?
      cmd = cmd_with_targets(
        ['terraform', 'destroy', '-force', "-var-file #{var_file_path}"],
        args[:target],
        args.extras
      )

      terraform_runner(cmd)
      @after_proc.call(t.name, @tf_dir) unless @after_proc.nil?
    end
  end
end
install_init() click to toggle source

add the 'tf:init' Rake task. This checks environment variables, runs “terraform -version“, and then runs “terraform init“ with the “backend_config“ options, if any.

# File lib/tfwrapper/raketasks.rb, line 166
def install_init
  namespace nsprefix do
    desc 'Run terraform init with appropriate arguments'
    task :init do |t|
      @before_proc.call(t.name, @tf_dir) unless @before_proc.nil?
      TFWrapper::Helpers.check_env_vars(
        @tf_vars_from_env.values, @allowed_empty_vars
      )
      @tf_version = check_tf_version
      cmd = [
        'terraform',
        'init',
        '-input=false'
      ].join(' ')
      @backend_config.each do |k, v|
        cmd = cmd + ' ' + "-backend-config='#{k}=#{v}'"
      end
      terraform_runner(cmd)
      @after_proc.call(t.name, @tf_dir) unless @after_proc.nil?
    end
  end
end
install_output() click to toggle source

add the 'tf:output' Rake task

# File lib/tfwrapper/raketasks.rb, line 265
def install_output
  namespace nsprefix do
    task output: [
      :"#{nsprefix}:init",
      :"#{nsprefix}:refresh"
    ] do |t|
      @before_proc.call(t.name, @tf_dir) unless @before_proc.nil?
      terraform_runner('terraform output')
      @after_proc.call(t.name, @tf_dir) unless @after_proc.nil?
    end
    task output_json: [
      :"#{nsprefix}:init",
      :"#{nsprefix}:refresh"
    ] do |t|
      @before_proc.call(t.name, @tf_dir) unless @before_proc.nil?
      terraform_runner('terraform output -json')
      @after_proc.call(t.name, @tf_dir) unless @after_proc.nil?
    end
  end
end
install_plan() click to toggle source

add the 'tf:plan' Rake task

# File lib/tfwrapper/raketasks.rb, line 190
def install_plan
  namespace nsprefix do
    desc 'Output the set plan to be executed by apply; specify ' \
      'optional CSV targets'
    task :plan, [:target] => [
      :"#{nsprefix}:init",
      :"#{nsprefix}:write_tf_vars"
    ] do |t, args|
      @before_proc.call(t.name, @tf_dir) unless @before_proc.nil?
      cmd = cmd_with_targets(
        ['terraform', 'plan', "-var-file #{var_file_path}"],
        args[:target],
        args.extras
      )

      stream_type = if HAVE_LANDSCAPE && !@disable_landscape
                      @landscape_progress
                    else
                      :stream
                    end
      outerr = terraform_runner(cmd, progress: stream_type)
      landscape_format(outerr) if HAVE_LANDSCAPE && !@disable_landscape
      @after_proc.call(t.name, @tf_dir) unless @after_proc.nil?
    end
  end
end
install_refresh() click to toggle source

add the 'tf:refresh' Rake task

# File lib/tfwrapper/raketasks.rb, line 245
def install_refresh
  namespace nsprefix do
    task refresh: [
      :"#{nsprefix}:init",
      :"#{nsprefix}:write_tf_vars"
    ] do |t|
      @before_proc.call(t.name, @tf_dir) unless @before_proc.nil?
      cmd = [
        'terraform',
        'refresh',
        "-var-file #{var_file_path}"
      ].join(' ')

      terraform_runner(cmd)
      @after_proc.call(t.name, @tf_dir) unless @after_proc.nil?
    end
  end
end
install_write_tf_vars() click to toggle source

add the 'tf:write_tf_vars' Rake task

# File lib/tfwrapper/raketasks.rb, line 317
def install_write_tf_vars
  namespace nsprefix do
    desc "Write #{var_file_path}"
    task :write_tf_vars do |t|
      @before_proc.call(t.name, @tf_dir) unless @before_proc.nil?
      tf_vars = terraform_vars
      puts 'Terraform vars:'
      tf_vars.sort.map do |k, v|
        redacted_list = (%w[aws_access_key aws_secret_key] +
                         @tf_sensitive_vars)
        if redacted_list.include?(k)
          puts "#{k} => (redacted)"
        else
          puts "#{k} => #{v}"
        end
      end
      File.open(var_file_path, 'w') do |f|
        f.write(tf_vars.to_json)
      end
      STDERR.puts "Terraform vars written to: #{var_file_path}"
      @after_proc.call(t.name, @tf_dir) unless @after_proc.nil?
    end
  end
end
landscape_format(output) click to toggle source

Given a string of terraform plan output, format it with terraform_landscape and print the result to STDOUT.

# File lib/tfwrapper/raketasks.rb, line 351
def landscape_format(output)
  p = TerraformLandscape::Printer.new(
    TerraformLandscape::Output.new(STDOUT)
  )
  p.process_string(output)
rescue StandardError, ScriptError => ex
  STDERR.puts 'Exception calling terraform_landscape to reformat ' \
              "output: #{ex.class.name}: #{ex}"
  puts output unless @landscape_progress == :stream
end
min_tf_version() click to toggle source
# File lib/tfwrapper/raketasks.rb, line 36
def min_tf_version
  Gem::Version.new('0.9.0')
end
nsprefix() click to toggle source
# File lib/tfwrapper/raketasks.rb, line 143
def nsprefix
  if @ns_prefix.nil?
    'tf'.to_sym
  else
    "#{@ns_prefix}_tf".to_sym
  end
end
terraform_runner(cmd, opts = {}) click to toggle source

Run a Terraform command, providing some useful output and handling AWS API rate limiting gracefully. Raises StandardError on failure. The command is run in @tf_dir.

@param cmd [String] Terraform command to run @option opts [Hash] :progress How to handle streaming output. Possible

values are ``:stream`` (default) to stream each line in STDOUT/STDERR
to STDOUT, ``:dots`` to print a dot for each line, ``:lines`` to print
a dot followed by a newline for each line, or ``nil`` to not stream any
output at all.

@return [String] combined STDOUT and STDERR rubocop:disable Metrics/PerceivedComplexity

# File lib/tfwrapper/raketasks.rb, line 374
def terraform_runner(cmd, opts = {})
  stream_type = opts.fetch(:progress, :stream)
  unless [:dots, :lines, :stream, nil].include?(stream_type)
    raise(
      ArgumentError,
      'progress option must be one of: [:dots, :lines, :stream, nil]'
    )
  end
  require 'retries'
  STDERR.puts "terraform_runner command: '#{cmd}' (in #{@tf_dir})"
  out_err = nil
  status = nil
  # exponential backoff as long as we're getting 403s
  handler = proc do |exception, attempt_number, total_delay|
    STDERR.puts "terraform_runner failed with #{exception}; retry " \
      "attempt #{attempt_number}; #{total_delay} seconds have passed."
  end
  status = -1
  with_retries(
    max_tries: 5,
    handler: handler,
    base_sleep_seconds: 1.0,
    max_sleep_seconds: 10.0
  ) do
    # this streams STDOUT and STDERR as a combined stream,
    # and also captures them as a combined string
    out_err, status = TFWrapper::Helpers.run_cmd_stream_output(
      cmd, @tf_dir, progress: stream_type
    )
    if status != 0 && out_err.include?('hrottling')
      raise StandardError, "#{out_err}\nTerraform hit AWS API rate limiting"
    end
    if status != 0 && out_err.include?('status code: 403')
      raise StandardError, "#{out_err}\nTerraform command got 403 error " \
        '- access denied or credentials not propagated'
    end
    if status != 0 && out_err.include?('status code: 401')
      raise StandardError, "#{out_err}\nTerraform command got 401 error " \
        '- access denied or credentials not propagated'
    end
  end
  # end exponential backoff
  unless status.zero?
    # if we weren't streaming output, send it now
    STDERR.puts out_err unless stream_type == :stream
    raise StandardError, "Errors have occurred executing: '#{cmd}' " \
      "(exited #{status})"
  end
  STDERR.puts "terraform_runner command '#{cmd}' finished and exited 0"
  out_err
end
terraform_vars() click to toggle source
# File lib/tfwrapper/raketasks.rb, line 342
def terraform_vars
  res = {}
  @tf_vars_from_env.each { |tfname, envname| res[tfname] = ENV[envname] }
  @tf_extra_vars.each { |name, val| res[name] = val }
  res
end
update_consul_stack_env_vars() click to toggle source

update stack status in Consul

# File lib/tfwrapper/raketasks.rb, line 459
def update_consul_stack_env_vars
  require 'diplomat'
  require 'json'
  data = {}
  @tf_vars_from_env.each_value { |k| data[k] = ENV[k] }

  Diplomat.configure do |config|
    config.url = @consul_url
  end

  puts "Writing stack information to #{@consul_url} at: "\
    "#{@consul_env_vars_prefix}"

  redacted_list = (%w[aws_access_key aws_secret_key] +
    @tf_sensitive_vars)
  sanitized_data = data.clone
  @tf_vars_from_env.each do |k, v|
    # We are trying to determine which ENV var maps
    #   to the sensitive terraform variable
    sanitized_data[v] = '(redacted)' if redacted_list.include?(k)
  end
  puts JSON.pretty_generate(sanitized_data)
  raw = JSON.generate(data)
  Diplomat::Kv.put(@consul_env_vars_prefix, raw)
end
var_file_path() click to toggle source
# File lib/tfwrapper/raketasks.rb, line 308
def var_file_path
  if @ns_prefix.nil?
    File.absolute_path('build.tfvars.json')
  else
    File.absolute_path("#{@ns_prefix}_build.tfvars.json")
  end
end