class MU::Config

Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.

Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.

Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.

Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.

Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.

Constants

CIDR_DESCRIPTION
CIDR_PATTERN
CIDR_PRIMITIVE

Attributes

config[R]
config_path[R]

The path to the most recently loaded configuration file

existing_deploy[R]
kittencfg_semaphore[R]
kittens[R]
nat_routes[RW]
parameters[R]
skipinitialupdates[R]
tails[R]
updating[R]

Public Class Methods

addDependency(resource, name, type, their_phase: "create", my_phase: nil) click to toggle source

Insert a dependency into the config hash of a resource, with sensible error checking and de-duplication. @param resource [Hash] @param name [String] @param type [String] @param phase [String] @param no_create_wait [Boolean]

# File modules/mu/config.rb, line 440
def self.addDependency(resource, name, type, their_phase: "create", my_phase: nil)
  if ![nil, "create", "groom"].include?(their_phase)
    raise MuError, "Invalid their_phase '#{their_phase}' while adding dependency #{type} #{name} to #{resource['name']}"
  end
  resource['dependencies'] ||= []
  _shortclass, cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(type)

  resource['dependencies'].each { |dep|
    if dep['type'] == cfg_name and dep['name'].to_s == name.to_s
      dep["their_phase"] = their_phase if their_phase
      dep["my_phase"] = my_phase if my_phase
      return
    end
  }

  newdep = {
    "type" => cfg_name,
    "name"  => name.to_s
  }
  newdep["their_phase"] = their_phase if their_phase
  newdep["my_phase"] = my_phase if my_phase

  resource['dependencies'] << newdep

end
cloud_primitive() click to toggle source

Configuration chunk for choosing a cloud provider @return [Hash]

# File modules/mu/config/schema_helpers.rb, line 163
    def self.cloud_primitive
      {
        "type" => "string",
#        "default" => MU::Config.defaultCloud, # applyInheritedDefaults does this better
        "enum" => MU::Cloud.supportedClouds
      }
    end
config_path() click to toggle source

The path to the most recently loaded configuration file

# File modules/mu/config.rb, line 49
def self.config_path
  @@config_path
end
credentials_primitive() click to toggle source

Configuration chunk for choosing a set of cloud credentials @return [Hash]

# File modules/mu/config/schema_helpers.rb, line 118
def self.credentials_primitive
  {
      "type" => "string",
      "description" => "Specify a non-default set of credentials to use when authenticating to cloud provider APIs, as listed in `mu.yaml` under each provider's subsection. If "
  }
end
defaultCloud() click to toggle source

The default cloud provider for new resources. Must exist in MU.supportedClouds return [String]

# File modules/mu/config/schema_helpers.rb, line 22
def self.defaultCloud
  configured = {}
  MU::Cloud.supportedClouds.each { |cloud|
    if $MU_CFG[cloud.downcase] and !$MU_CFG[cloud.downcase].empty?
      configured[cloud] = $MU_CFG[cloud.downcase].size
      configured[cloud] += 0.5 if MU::Cloud.cloudClass(cloud).hosted? # tiebreaker
    end
  }
  if configured.size > 0
    return configured.keys.sort { |a, b|
      configured[b] <=> configured[a]
    }.first
  else
    MU::Cloud.supportedClouds.each { |cloud|
      return cloud if MU::Cloud.cloudClass(cloud).hosted?
    }
    return MU::Cloud.supportedClouds.first
  end
end
defaultGroomer() click to toggle source

The default grooming agent for new resources. Must exist in MU.supportedGroomers.

# File modules/mu/config/schema_helpers.rb, line 43
def self.defaultGroomer
  MU.localOnly ? "Ansible" : "Chef"
end
dependencies_primitive() click to toggle source

JSON-schema for resource dependencies @return [Hash]

# File modules/mu/config/schema_helpers.rb, line 174
def self.dependencies_primitive
  {
    "type" => "array",
    "items" => {
      "type" => "object",
      "description" => "Declare other objects which this resource requires. This resource will wait until the others are available to create itself.",
      "required" => ["name", "type"],
      "additionalProperties" => false,
      "properties" => {
        "name" => {"type" => "string"},
        "type" => {
          "type" => "string",
          "enum" => MU::Cloud.resource_types.values.map { |v| v[:cfg_name] }
        },
        "my_phase" => {
          "type" => "string",
          "description" => "Which part of our creation process should be waiting?",
          "enum" => ["create", "groom"]
        },
        "their_phase" => {
          "type" => "string",
          "description" => "Which part of the creation process of the resource we depend on should we wait for before starting our own creation? Defaults are usually sensible, but sometimes you want, say, a Server to wait on another Server to be completely ready (through its groom phase) before starting up.",
          "enum" => ["create", "groom"]
        },
        "phase" => {
          "type" => "string",
          "description" => "Alias for {their_phase}",
          "enum" => ["create", "groom"]
        },
        "no_create_wait" => {
          "type" => "boolean",
          "description" => "DEPRECATED- setting +true+ is the same as setting {my_phase} to +groom+; setting to +false+ is the same as setting {my_phase} to +create+. If both +no_create_wait+ and {my_phase} are specified, {my_phase} takes precedence."
        }
      }
    }
  }
end
docSchema() click to toggle source

Accessor for our Basket of Kittens schema definition, with various cloud-specific details merged so we can generate documentation for them.

# File modules/mu/config/doc_helpers.rb, line 22
def self.docSchema
  docschema = Marshal.load(Marshal.dump(@@schema))
  only_children = {}
  MU::Cloud.resource_types.each_pair { |classname, attrs|
    MU::Cloud.supportedClouds.each { |cloud|
      begin
        require "mu/providers/#{cloud.downcase}/#{attrs[:cfg_name]}"
      rescue LoadError
        next
      end
      _required, res_schema = MU::Cloud.resourceClass(cloud, classname).schema(self)
      docschema["properties"][attrs[:cfg_plural]]["items"]["description"] ||= ""
      docschema["properties"][attrs[:cfg_plural]]["items"]["description"] += "\n#\n# `#{cloud}`: "+MU::Cloud.resourceClass(cloud, classname).quality
      res_schema.each { |key, cfg|
        if !docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
          only_children[attrs[:cfg_plural]] ||= {}
          only_children[attrs[:cfg_plural]][key] ||= {}
          only_children[attrs[:cfg_plural]][key][cloud] = cfg
        end
      }
    }
  }

  # recursively chase down description fields in arrays and objects of our
  # schema and prepend stuff to them for documentation
  def self.prepend_descriptions(prefix, cfg)
    cfg["prefix"] = prefix
    if cfg["type"] == "array" and cfg["items"]
      cfg["items"] = prepend_descriptions(prefix, cfg["items"])
    elsif cfg["type"] == "object" and cfg["properties"]
      cfg["properties"].keys.each { |key|
        cfg["properties"][key] = prepend_descriptions(prefix, cfg["properties"][key])
      }
    end
    cfg
  end

  MU::Cloud.resource_types.each_pair { |classname, attrs|
    MU::Cloud.supportedClouds.each { |cloud|
      res_class = nil
      begin
        res_class = MU::Cloud.resourceClass(cloud, classname)
      rescue MU::Cloud::MuCloudResourceNotImplemented
        next
      end
      required, res_schema = res_class.schema(self)
      next if required.size == 0 and res_schema.size == 0
      res_schema.each { |key, cfg|
        cfg["description"] ||= ""
        if !cfg["description"].empty?
          cfg["description"] = "\n# +"+cloud.upcase+"+: "+cfg["description"]
        end
        if docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
          schemaMerge(docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key], cfg, cloud)
          docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] ||= ""
          docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] += "\n"+(cfg["description"].match(/^#/) ? "" : "# ")+cfg["description"]
          MU.log "Munging #{cloud}-specific #{classname.to_s} schema into BasketofKittens => #{attrs[:cfg_plural]} => #{key}", MU::DEBUG, details: docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
        else
          if only_children[attrs[:cfg_plural]][key]
            prefix = only_children[attrs[:cfg_plural]][key].keys.map{ |x| x.upcase }.join(" & ")+" ONLY"
            cfg["description"].gsub!(/^\n#/, '') # so we don't leave the description blank in the "optional parameters" section
            cfg = prepend_descriptions(prefix, cfg)
          end

          docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key] = cfg
        end
        docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["clouds"] = {}
        docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["clouds"][cloud] = cfg
      }

      docschema['required'].concat(required)
      docschema['required'].uniq!
    }
  }

  docschema
end
emitConfigAsRuby() click to toggle source

Generate a documentation-friendly dummy Ruby class for our mu.yaml main config.

# File modules/mu/config/doc_helpers.rb, line 141
    def self.emitConfigAsRuby
      example = %Q{---
public_address: 1.2.3.4
mu_admin_email: egtlabs@eglobaltech.com
mu_admin_name: Joe Schmoe
mommacat_port: 2260
banner: My Example Mu Master
mu_repository: git://github.com/cloudamatic/mu.git
repos:
- https://github.com/cloudamatic/mu_demo_platform
allow_invade_foreign_vpcs: true
ansible_dir:
aws:
  egtdev:
    region: us-east-1
    log_bucket_name: egt-mu-log-bucket
    default: true
    name: egtdev
  personal:
    region: us-east-2
    log_bucket_name: my-mu-log-bucket
    name: personal
  google:
    egtlabs:
      project: egt-labs-admin
      credentials_file: /opt/mu/etc/google.json
      region: us-east4
      log_bucket_name: hexabucket-761234
      default: true
}
      mu_yaml_schema = eval(%Q{
$NOOP = true
load "#{MU.myRoot}/bin/mu-configure"
$CONFIGURABLES
})
      return if mu_yaml_schema.nil? or !mu_yaml_schema.is_a?(Hash)
      muyamlpath = "#{MU.myRoot}/modules/mu/mu.yaml.rb"
      MU.log "Converting mu.yaml schema to Ruby objects in #{muyamlpath}"
      muyaml_rb = File.new(muyamlpath, File::CREAT|File::TRUNC|File::RDWR, 0644)
      muyaml_rb.puts "# Configuration schema for mu.yaml. See also {https://github.com/cloudamatic/mu/wiki/Configuration the Mu wiki}."
      muyaml_rb.puts "#"
      muyaml_rb.puts "# Example:"
      muyaml_rb.puts "#"
      muyaml_rb.puts "# <pre>"
      example.split(/\n/).each { |line|
        muyaml_rb.puts "#      "+line+"    " # markdooooown
      }
      muyaml_rb.puts "# </pre>"
      muyaml_rb.puts "module MuYAML"
      muyaml_rb.puts "\t# The configuration file format for Mu's main config file."
      MU::Config.printMuYamlSchema(muyaml_rb, [], { "subtree" => mu_yaml_schema })
      muyaml_rb.puts "end"
      muyaml_rb.close
    end
emitSchemaAsRuby() click to toggle source

Take the schema we've defined and create a dummy Ruby class tree out of it, basically so we can leverage Yard to document it.

# File modules/mu/config/doc_helpers.rb, line 198
def self.emitSchemaAsRuby
  kittenpath = "#{MU.myRoot}/modules/mu/kittens.rb"
  MU.log "Converting Basket of Kittens schema to Ruby objects in #{kittenpath}"
  kitten_rb = File.new(kittenpath, File::CREAT|File::TRUNC|File::RDWR, 0644)
  kitten_rb.puts "### THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT ###"
  kitten_rb.puts "#"
  kitten_rb.puts "#"
  kitten_rb.puts "#"
  kitten_rb.puts "module MU"
  kitten_rb.puts "class Config"
  kitten_rb.puts "\t# The configuration file format for Mu application stacks."
  self.printSchema(kitten_rb, ["BasketofKittens"], MU::Config.docSchema)
  kitten_rb.puts "end"
  kitten_rb.puts "end"
  kitten_rb.close

end
fixDashes(conf) click to toggle source

We used to be inconsistent about config keys using dashes versus underscores. Now we've standardized on the latter. Be polite and translate for older configs, since we're not fussed about name collisions.

# File modules/mu/config.rb, line 1047
def self.fixDashes(conf)
  if conf.is_a?(Hash)
    newhash = Hash.new
    conf.each_pair { |key, val|
      if val.is_a?(Hash) or val.is_a?(Array)
        val = self.fixDashes(val)
      end
      if key.match(/-/)
        MU.log "Replacing #{key} with #{key.gsub(/-/, "_")}", MU::DEBUG
        newhash[key.gsub(/-/, "_")] = val
      else
        newhash[key] = val
      end
    }
    return newhash
  elsif conf.is_a?(Array)
    conf.map! { |val|
      if val.is_a?(Hash) or val.is_a?(Array)
        self.fixDashes(val)
      else
        val
      end
    }
  end

  return conf
end
global_bindings() click to toggle source

Keep a cache of bindings we've created as sandbox contexts for ERB processing, so we don't keep reloading the entire Mu library inside new ones.

# File modules/mu/config.rb, line 1147
def self.global_bindings
  @@bindings
end
guessFormat(path) click to toggle source

Given a path to a config file, try to guess whether it's YAML or JSON. @param path [String]: The path to the file to check.

# File modules/mu/config.rb, line 1004
def self.guessFormat(path)
  raw = File.read(path)
  # Rip out ERB references that will bollocks parser syntax, first.
  stripped = raw.gsub(/<%.*?%>,?/, "").gsub(/,[\n\s]*([\]\}])/, '\1')
  begin
    JSON.parse(stripped)
  rescue JSON::ParserError
    begin
      YAML.load(raw.gsub(/<%.*?%>/, ""))
    rescue Psych::SyntaxError
      # Ok, well neither of those worked, let's assume that filenames are
      # meaningful.
      if path.match(/\.(yaml|yml)$/i)
        MU.log "Guessing that #{path} is YAML based on filename", MU::DEBUG
        return :yaml
      elsif path.match(/\.(json|jsn|js)$/i)
        MU.log "Guessing that #{path} is JSON based on filename", MU::DEBUG
        return :json
      else
        # For real? Ok, let's try the dumbest possible method.
        dashes = raw.match(/\-/)
        braces = raw.match(/[{}]/)
        if dashes.size > braces.size
          MU.log "Guessing that #{path} is YAML by... counting dashes.", MU::NOTICE
          return :yaml
        elsif braces.size > dashes.size
          MU.log "Guessing that #{path} is JSON by... counting braces.", MU::NOTICE
          return :json
        else
          raise "Unable to guess composition of #{path} by any means"
        end
      end
    end
    MU.log "Guessing that #{path} is YAML based on parser", MU::DEBUG
    return :yaml
  end
  MU.log "Guessing that #{path} is JSON based on parser", MU::NOTICE
  return :json
end
include(file, binding = nil, param_pass = false) click to toggle source

This can be called with ERB from within a stack config file, like so: <%= Config.include(“drupal.json”) %> It will first try the literal path you pass it, and if it fails to find that it will look in the directory containing the main (top-level) config.

# File modules/mu/config.rb, line 1081
    def self.include(file, binding = nil, param_pass = false)
      loglevel = param_pass ? MU::NOTICE : MU::DEBUG
      retries = 0
      orig_filename = file
      assume_type = nil
      if file.match(/(js|json|jsn)$/i)
        assume_type = :json
      elsif file.match(/(yaml|yml)$/i)
        assume_type = :yaml
      end
      begin
        erb = ERB.new(File.read(file), nil, "<>")
      rescue Errno::ENOENT
        retries = retries + 1
        if retries == 1
          file = File.dirname(MU::Config.config_path)+"/"+orig_filename
          retry
        elsif retries == 2
          file = File.dirname(MU.myRoot)+"/lib/demo/"+orig_filename
          retry
        else
          raise ValidationError, "Couldn't read #{file} included from #{MU::Config.config_path}"
        end
      end
      begin
        # Include as just a drop-in block of text if the filename doesn't imply
        # a particular format, or if we're melding JSON into JSON.
        if ($file_format == :json and assume_type == :json) or assume_type.nil?
          MU.log "Including #{file} as uninterpreted text", loglevel
          return erb.result(binding)
        end
        # ...otherwise, try to parse into something useful so we can meld
        # differing file formats, or work around YAML's annoying dependence
        # on indentation.
        parsed_cfg = nil
        begin
          parsed_cfg = JSON.parse(erb.result(binding))
#          parsed_as = :json
        rescue JSON::ParserError => e
          MU.log e.inspect, MU::DEBUG
          begin
            parsed_cfg = YAML.load(MU::Config.resolveYAMLAnchors(erb.result(binding)))
#            parsed_as = :yaml
          rescue Psych::SyntaxError => e
            MU.log e.inspect, MU::DEBUG
            MU.log "#{file} parsed neither as JSON nor as YAML, including as raw text", MU::WARN if @param_pass
            return erb.result(binding)
          end
        end
        if $file_format == :json
          MU.log "Including #{file} as interpreted JSON", loglevel
          return JSON.generate(parsed_cfg)
        else
          MU.log "Including #{file} as interpreted YAML", loglevel
          $yaml_refs[file] = ""+YAML.dump(parsed_cfg).sub(/^---\n/, "")
          return "# MU::Config.include PLACEHOLDER #{file} REDLOHECALP"
        end
      rescue SyntaxError
        raise ValidationError, "ERB in #{file} threw a syntax error"
      end
    end
loadResourceSchema(type, cloud: nil) click to toggle source

Load and validate the schema for an individual resource class, optionally merging cloud-specific schema components. @param type [String]: The resource type to load @param cloud [String]: A specific cloud, whose implementation's schema of this resource we will merge @return [Hash]

# File modules/mu/config/schema_helpers.rb, line 228
def self.loadResourceSchema(type, cloud: nil)
  valid = true
  shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(type)
  schemaclass = Object.const_get("MU").const_get("Config").const_get(shortclass)

  [:schema, :validate].each { |method|
    if !schemaclass.respond_to?(method)
      MU.log "MU::Config::#{type}.#{method.to_s} doesn't seem to be implemented", MU::ERR
      return [nil, false] if method == :schema
      valid = false
    end
  }

  schema = schemaclass.schema.dup

  schema["properties"]["virtual_name"] = {
    "description" => "Internal use.",
    "type" => "string"
  }
  schema["properties"]["dependencies"] = MU::Config.dependencies_primitive
  schema["properties"]["cloud"] = MU::Config.cloud_primitive
  schema["properties"]["credentials"] = MU::Config.credentials_primitive
  schema["title"] = type.to_s

  if cloud
    cloudclass = MU::Cloud.resourceClass(cloud, type)

    if cloudclass.respond_to?(:schema)
      _reqd, cloudschema = cloudclass.schema
      cloudschema.each { |key, cfg|
        if schema["properties"][key]
          schemaMerge(schema["properties"][key], cfg, cloud)
        else
          schema["properties"][key] = cfg.dup
        end
      }
    else
      MU.log "MU::Cloud::#{cloud}::#{type}.#{method.to_s} doesn't seem to be implemented", MU::ERR
      valid = false
    end

  end

  return [schema, valid]
end
manxify(config, remove_runtime_keys: false) click to toggle source

Run through a config hash and return a version with all {MU::Config::Tail} endpoints converted to plain strings. Useful for cloud layers that don't care about the metadata in Tails. @param config [Hash]: The configuration tree to convert @return [Hash]: The modified configuration

# File modules/mu/config.rb, line 76
def self.manxify(config, remove_runtime_keys: false)
  if config.is_a?(Hash)
    newhash = {}
    config.each_pair { |key, val|
      next if remove_runtime_keys and (key.nil? or key.match(/^#MU_/))
      next if val.is_a?(Array) and val.empty?
      newhash[key] = self.manxify(val, remove_runtime_keys: remove_runtime_keys)
    }
    config = newhash
  elsif config.is_a?(Array)
    newarray = []
    config.each { |val|
      newarray << self.manxify(val, remove_runtime_keys: remove_runtime_keys)
    }
    config = newarray
  elsif config.is_a?(MU::Config::Tail)
    return config.to_s
  elsif config.is_a?(MU::Config::Ref)
    return self.manxify(config.to_h,  remove_runtime_keys: remove_runtime_keys)
  end
  return config
end
new(path, skipinitialupdates = false, params: {}, updating: nil, default_credentials: nil, cloud: nil) click to toggle source

Load, resolve, and validate a configuration file (“Basket of Kittens”). @param path [String]: The path to the master config file to load. Note that this can include other configuration files via ERB. @param skipinitialupdates [Boolean]: Whether to forcibly apply the skipinitialupdates flag to nodes created by this configuration. @param params [Hash]: Optional name-value parameter pairs, which will be passed to our configuration files as ERB variables. @param cloud [String]: Sets a parameter named 'cloud', and insert it as the default cloud platform if not already declared @return [Hash]: The complete validated configuration for a deployment.

# File modules/mu/config.rb, line 257
    def initialize(path, skipinitialupdates = false, params: {}, updating: nil, default_credentials: nil, cloud: nil)
      $myPublicIp ||= MU.mu_public_ip
      $myRoot ||= MU.myRoot
      $myRoot.freeze

      $myAZ ||= MU.myAZ.freeze
      $myAZ.freeze
      $myRegion ||= MU.curRegion.freeze
      $myRegion.freeze
      
      @kittens = {}
      @kittencfg_semaphore = Mutex.new
      @@config_path = path
      @admin_firewall_rules = []
      @skipinitialupdates = skipinitialupdates
      @updating = updating
      if @updating
        @existing_deploy = MU::MommaCat.new(@updating)
      end
      @default_credentials = default_credentials

      ok = true
      params.each_pair { |name, value|
        begin
          raise DeployParamError, "Parameter must be formatted as name=value" if value.nil? or value.empty?
          raise DeployParamError, "Parameter name must be a legal Ruby variable name" if name.match(/[^A-Za-z0-9_]/)
          raise DeployParamError, "Parameter values cannot contain quotes" if value.match(/["']/)
          eval("defined? $#{name} and raise DeployParamError, 'Parameter name reserved'")
          @@parameters[name] = value
          @@user_supplied_parameters[name] = value
          eval("$#{name}='#{value}'") # support old-style $global parameter refs
          MU.log "Passing variable $#{name} into #{@@config_path} with value '#{value}'"
        rescue RuntimeError, SyntaxError => e
          ok = false
          MU.log "Error setting $#{name}='#{value}': #{e.message}", MU::ERR
        end
      }

      if cloud and !@@parameters["cloud"]
        if !MU::Cloud.availableClouds.include?(cloud)
          ok = false
          MU.log "Provider '#{cloud}' is not listed as an available cloud", MU::ERR, details: MU::Cloud.availableClouds
        else
          @@parameters["cloud"] = getTail("cloud", value: cloud, pseudo: true)
          @@user_supplied_parameters["cloud"] = cloud
          eval("$cloud='#{cloud}'") # support old-style $global parameter refs
        end
      end
      raise ValidationError if !ok

      # Run our input through the ERB renderer, a first pass just to extract
      # the parameters section so that we can resolve all of those to variables
      # for the rest of the config to reference.
      # XXX Figure out how to make include() add parameters for us. Right now
      # you can't specify parameters in an included file, because ERB is what's
      # doing the including, and parameters need to already be resolved so that
      # ERB can use them.
      param_cfg, _raw_erb_params_only = resolveConfig(path: @@config_path, param_pass: true, cloud: cloud)
      if param_cfg.has_key?("parameters")
        param_cfg["parameters"].each { |param|
          if param.has_key?("default") and param["default"].nil?
            param["default"] = ""
          end
        }
      end

      # Set up special Tail objects for our automatic pseudo-parameters
      getTail("myPublicIp", value: $myPublicIp, pseudo: true)
      getTail("myRoot", value: $myRoot, pseudo: true)
      getTail("myAZ", value: $myAZ, pseudo: true)
      getTail("myRegion", value: $myRegion, pseudo: true)

      if param_cfg.has_key?("parameters") and !param_cfg["parameters"].nil? and param_cfg["parameters"].size > 0
        param_cfg["parameters"].each { |param|
          param['valid_values'] ||= []
          if !@@parameters.has_key?(param['name'])
            if param.has_key?("default")
              @@parameters[param['name']] = param['default'].nil? ? "" : param['default']
            elsif param["required"] or !param.has_key?("required")
              MU.log "Required parameter '#{param['name']}' not supplied", MU::ERR
              ok = false
              next
            else # not required, no default
              next
            end
          end
          if param.has_key?("cloudtype")
            getTail(param['name'], value: @@parameters[param['name']], cloudtype: param["cloudtype"], valid_values: param['valid_values'], description: param['description'], prettyname: param['prettyname'], list_of: param['list_of'])
          else
            getTail(param['name'], value: @@parameters[param['name']], valid_values: param['valid_values'], description: param['description'], prettyname: param['prettyname'], list_of: param['list_of'])
          end
        }
      end

      raise ValidationError if !ok
      @@parameters.each_pair { |name, val|
        next if @@tails.has_key?(name) and @@tails[name].is_a?(MU::Config::Tail) and @@tails[name].pseudo
        # Parameters can have limited parameterization of their own
        if @@tails[name].to_s.match(/^(.*?)MU::Config.getTail PLACEHOLDER (.+?) REDLOHECALP(.*)/)
          @@tails[name] = getTail(name, value: @@tails[$2])
        end

        if respond_to?(name.to_sym)
          MU.log "Parameter name '#{name}' reserved", MU::ERR
          ok = false
          next
        end
        MU.log "Passing variable '#{name}' into #{path} with value '#{val}'"
      }
      raise DeployParamError, "One or more invalid parameters specified" if !ok
      $parameters = @@parameters.dup
      $parameters.freeze

      tmp_cfg, _raw_erb = resolveConfig(path: @@config_path, cloud: cloud)

      # Convert parameter entries that constitute whole config keys into
      # {MU::Config::Tail} objects.
      def resolveTails(tree, indent= "")
        if tree.is_a?(Hash)
          tree.each_pair { |key, val|
            tree[key] = resolveTails(val, indent+" ")
          }
        elsif tree.is_a?(Array)
          newtree = []
          tree.each { |item|
            newtree << resolveTails(item, indent+" ")
          }
          tree = newtree
        elsif tree.is_a?(String) and tree.match(/^(.*?)MU::Config.getTail PLACEHOLDER (.+?) REDLOHECALP(.*)/)
          tree = getTail($2, prefix: $1, suffix: $3)
          if tree.nil? and @@tails.has_key?($2) # XXX why necessary?
            tree = @@tails[$2]
          end
        end
        return tree
      end
      @config = resolveTails(tmp_cfg)
      @config.merge!(param_cfg)

      if !@config.has_key?('admins') or @config['admins'].size == 0
        @config['admins'] = [
          {
            "name" => MU.chef_user == "mu" ? "Mu Administrator" : MU.userName,
            "email" => MU.userEmail
          }
        ]
      end

      @config['credentials'] ||= @default_credentials

      if @config['cloud'] and !MU::Cloud.availableClouds.include?(@config['cloud'])
        if MU::Cloud.supportedClouds.include?(@config['cloud'])
          MU.log "Cloud provider #{@config['cloud']} declared, but no #{@config['cloud']} credentials available", MU::ERR
        else
          MU.log "Cloud provider #{@config['cloud']} is not supported", MU::ERR, details: MU::Cloud.supportedClouds
        end
        exit 1
      end

      MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }.each { |type|
        if @config[type]
          @config[type].each { |k|
            next if !k.is_a?(Hash)
            applyInheritedDefaults(k, type)
          }
        end
      }
      applySchemaDefaults(@config, MU::Config.schema)

      validate # individual resources validate when added now, necessary because the schema can change depending on what cloud they're targeting
#      XXX but now we're not validating top-level keys, argh
#pp @config
#raise "DERP"
      @config.freeze
    end
notification_email() click to toggle source

Have a default value available for config schema elements that take an email address. @return [String]

# File modules/mu/config/schema_helpers.rb, line 215
def self.notification_email 
  if MU.chef_user == "mu"
    ENV['MU_ADMIN_EMAIL']
  else
    MU.userEmail
  end
end
optional_tags_primitive() click to toggle source

Configuration chunk for creating resource tags as an array of key/value pairs. @return [Hash]

# File modules/mu/config/schema_helpers.rb, line 128
def self.optional_tags_primitive
  {
    "type" => "boolean",
    "description" => "Tag the resource with our optional tags (+MU-HANDLE+, +MU-MASTER-NAME+, +MU-OWNER+).",
    "default" => true
  }
end
parameters() click to toggle source

Accessor for parameters to our Basket of Kittens

# File modules/mu/config.rb, line 59
def self.parameters
  @@parameters
end
prepend_descriptions(prefix, cfg) click to toggle source

recursively chase down description fields in arrays and objects of our schema and prepend stuff to them for documentation

# File modules/mu/config/doc_helpers.rb, line 47
def self.prepend_descriptions(prefix, cfg)
  cfg["prefix"] = prefix
  if cfg["type"] == "array" and cfg["items"]
    cfg["items"] = prepend_descriptions(prefix, cfg["items"])
  elsif cfg["type"] == "object" and cfg["properties"]
    cfg["properties"].keys.each { |key|
      cfg["properties"][key] = prepend_descriptions(prefix, cfg["properties"][key])
    }
  end
  cfg
end
printMuYamlSchema(muyaml_rb, class_hierarchy, schema, in_array = false, required = false) click to toggle source

Emit our mu.yaml schema in a format that YARD can comprehend and turn into documentation.

# File modules/mu/config/doc_helpers.rb, line 390
    def self.printMuYamlSchema(muyaml_rb, class_hierarchy, schema, in_array = false, required = false)
      return if schema.nil?
      if schema["subtree"]
        printme = Array.new
        # order sub-elements by whether they're required, so we can use YARD's
        # grouping tags on them
        have_required = schema["subtree"].keys.any? { |k| schema["subtree"][k]["required"] }
        prop_list = schema["subtree"].keys.sort { |a, b|
          if schema["subtree"][a]["required"] and !schema["subtree"][b]["required"]
            -1
          elsif !schema["subtree"][a]["required"] and schema["subtree"][b]["required"]
            1
          else
            a <=> b
          end
        }

        req = false
        printme << "# @!group Optional parameters" if !have_required
        prop_list.each { |name|
          prop = schema["subtree"][name]
          if prop["required"]
            printme << "# @!group Required parameters" if !req
            req = true
          else
            if req
              printme << "# @!endgroup"
              printme << "# @!group Optional parameters"
            end
            req = false
          end

          printme << self.printMuYamlSchema(muyaml_rb, class_hierarchy+ [name], prop, false, req)
        }
        printme << "# @!endgroup"

        desc = (schema['desc'] || schema['title'])

        tabs = 1
        class_hierarchy.each { |classname|
          if classname == class_hierarchy.last and desc
            muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{desc}\n"
          end
          muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}"
          tabs = tabs + 1
        }
        printme.each { |lines|
          if !lines.nil? and lines.is_a?(String)
            lines.lines.each { |line|
              muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + line
            }
          end
        }

#        class_hierarchy.each { |classname|
#          tabs = tabs - 1
#          muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end"
#        }
        i = class_hierarchy.size
        until i == 0 do
          tabs = tabs - 1
          muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end"
          i -= 1
        end

        # And now that we've dealt with our children, pass our own rendered
        # commentary back up to our caller.
        name = class_hierarchy.last
        if in_array
          type = "Array<#{class_hierarchy.join("::")}>"
        else
          type = class_hierarchy.join("::")
        end

        docstring = "\n"
        docstring = docstring + "# **REQUIRED**\n" if required
#        docstring = docstring + "# **"+schema["prefix"]+"**\n" if schema["prefix"]
        docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if desc
        docstring = docstring + "#\n"
        docstring = docstring + "# @return [#{type}]\n"
        docstring = docstring + "# @see #{class_hierarchy.join("::")}\n"
        docstring = docstring + "attr_accessor :#{name}"
        return docstring

      else
        in_array = schema["array"]
        name = class_hierarchy.last
        type = if schema['boolean']
          "Boolean"
        else
          "String"
        end
        if in_array
          type = "Array<#{type}>"
        end
        docstring = "\n"

        prefixes = []
        prefixes << "# **REQUIRED**" if schema["required"] and schema['default'].nil?
#        prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"]
        prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil?
        if !schema['pattern'].nil?
          # XXX unquoted regex chars confuse the hell out of YARD. How do we
          # quote {}[] etc in YARD-speak?
          prefixes << "# **Must match pattern `#{schema['pattern'].to_s.gsub(/\n/, "\n#")}`**"
        end

        desc = (schema['desc'] || schema['title'])
        if prefixes.size > 0
          docstring += prefixes.join(",\n")
          if desc and desc.size > 1
            docstring += " - "
          end
          docstring += "\n"
        end

        docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if !desc.nil?
        docstring = docstring + "#\n"
        docstring = docstring + "# @return [#{type}]\n"
        docstring = docstring + "attr_accessor :#{name}"

        return docstring
      end
    end
printSchema(kitten_rb, class_hierarchy, schema, in_array = false, required = false, prefix: nil) click to toggle source

Emit our Basket of Kittens schema in a format that YARD can comprehend and turn into documentation.

# File modules/mu/config/doc_helpers.rb, line 218
def self.printSchema(kitten_rb, class_hierarchy, schema, in_array = false, required = false, prefix: nil)
  return if schema.nil?

  if schema["type"] == "object"
    printme = []

    if !schema["properties"].nil?
      # order sub-elements by whether they're required, so we can use YARD's
      # grouping tags on them
      if !schema["required"].nil? and schema["required"].size > 0
        prop_list = schema["properties"].keys.sort_by { |name|
          schema["required"].include?(name) ? 0 : 1
        }
      else
        prop_list = schema["properties"].keys
      end
      req = false
      printme << "# @!group Optional parameters" if schema["required"].nil? or schema["required"].size == 0
      prop_list.each { |name|
        prop = schema["properties"][name]

        if class_hierarchy.size == 1

          _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(name, false)
          if cfg_name
            example_path = MU.myRoot+"/modules/mu/config/"+cfg_name+".yml"
            if File.exist?(example_path)
              example = "#\n# Examples:\n#\n"
              # XXX these variables are all parameters from the BoKs in
              # modules/tests. A really clever implementation would read
              # and parse them to get default values, perhaps, instead of
              # hard-coding them here.
              instance_type = "t2.medium"
              db_size = "db.t2.medium"
              vpc_name = "some_vpc"
              logs_name = "some_loggroup"
              queues_name = "some_queue"
              server_pools_name = "some_server_pool"
              ["simple", "complex"].each { |complexity|
                erb = ERB.new(File.read(example_path), nil, "<>")
                example += "#      !!!yaml\n"
                example += "#      ---\n"
                example += "#      appname: #{complexity}\n"
                example += "#      #{cfg_plural}:\n"
                firstline = true
                erb.result(binding).split(/\n/).each { |l|
                  l.chomp!
                  l.sub!(/#.*/, "") if !l.match(/#(?:INTERNET|NAT|DENY)/)
                  next if l.empty? or l.match(/^\s+$/)
                  if firstline
                    l = "- "+l
                    firstline = false
                  else
                    l = "  "+l
                  end
                  example += "#      "+l+"    "+"\n"
                }
                example += "# &nbsp;\n#\n" if complexity == "simple"
              }
              schema["properties"][name]["items"]["description"] ||= ""
              if !schema["properties"][name]["items"]["description"].empty?
                schema["properties"][name]["items"]["description"] += "\n"
              end
              schema["properties"][name]["items"]["description"] += example
            end
          end
        end

        if !schema["required"].nil? and schema["required"].include?(name)
          printme << "# @!group Required parameters" if !req
          req = true
        else
          if req
            printme << "# @!endgroup"
            printme << "# @!group Optional parameters"
          end
          req = false
        end

        printme << self.printSchema(kitten_rb, class_hierarchy+ [name], prop, false, req, prefix: schema["prefix"])
      }
      printme << "# @!endgroup"
    end

    tabs = 1
    class_hierarchy.each { |classname|
      if classname == class_hierarchy.last and !schema['description'].nil?
        kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{schema['description']}\n"
      end
      kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}"
      tabs = tabs + 1
    }
    printme.each { |lines|
      if !lines.nil? and lines.is_a?(String)
        lines.lines.each { |line|
          kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + line
        }
      end
    }

    i = class_hierarchy.size
    until i == 0 do
      tabs = tabs - 1
      kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end"
      i -= 1
    end

    # And now that we've dealt with our children, pass our own rendered
    # commentary back up to our caller.
    name = class_hierarchy.last
    if in_array
      type = "Array<#{class_hierarchy.join("::")}>"
    else
      type = class_hierarchy.join("::")
    end

    docstring = "\n"
    docstring = docstring + "# **REQUIRED**\n" if required
    docstring = docstring + "# **"+schema["prefix"]+"**\n" if schema["prefix"]
    docstring = docstring + "# #{schema['description'].gsub(/\n/, "\n#")}\n" if !schema['description'].nil?
    docstring = docstring + "#\n"
    docstring = docstring + "# @return [#{type}]\n"
    docstring = docstring + "# @see #{class_hierarchy.join("::")}\n"
    docstring = docstring + "attr_accessor :#{name}"
    return docstring

  elsif schema["type"] == "array"
    return self.printSchema(kitten_rb, class_hierarchy, schema['items'], true, required, prefix: prefix)
  else
    name = class_hierarchy.last
    if schema['type'].nil?
      MU.log "Couldn't determine schema type in #{class_hierarchy.join(" => ")}", MU::WARN, details: schema
      return nil
    end
    if in_array
      type = "Array<#{schema['type'].capitalize}>"
    else
      type = schema['type'].capitalize
    end
    docstring = "\n"

    prefixes = []
    prefixes << "# **REQUIRED**" if required and schema['default'].nil?
    prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"]
    prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil?
    if !schema['enum'].nil? and !schema["enum"].empty?
      prefixes << "# **Must be one of: `#{schema['enum'].join(', ')}`**"
    elsif !schema['pattern'].nil?
      # XXX unquoted regex chars confuse the hell out of YARD. How do we
      # quote {}[] etc in YARD-speak?
      prefixes << "# **Must match pattern `#{schema['pattern'].gsub(/\n/, "\n#")}`**"
    end

    if prefixes.size > 0
      docstring += prefixes.join(",\n")
      if schema['description'] and schema['description'].size > 1
        docstring += " - "
      end
      docstring += "\n"
    end

    docstring = docstring + "# #{schema['description'].gsub(/\n/, "\n#")}\n" if !schema['description'].nil?
    docstring = docstring + "#\n"
    docstring = docstring + "# @return [#{type}]\n"
    docstring = docstring + "attr_accessor :#{name}"

    return docstring
  end
end
region_primitive() click to toggle source

Configuration chunk for choosing a provider region @return [Hash]

# File modules/mu/config/schema_helpers.rb, line 95
def self.region_primitive
  if !@@allregions or @@allregions.empty?
    @@allregions = []
    MU::Cloud.availableClouds.each { |cloud|
      next if @@loadfails.include?(cloud)
      cloudclass = MU::Cloud.cloudClass(cloud)
      begin
        return @@allregions if !cloudclass.listRegions()
        @@allregions.concat(cloudclass.listRegions())
      rescue MU::MuError => e
        @@loadfails << cloud
        MU.log e.message, MU::WARN
      end
    }
  end
  {
    "type" => "string",
    "enum" => @@allregions
  }
end
resolveYAMLAnchors(lines) click to toggle source

Ugly text-manipulation to recursively resolve some placeholder strings we put in for ERB include() directives. @param lines [String] @return [String]

# File modules/mu/config.rb, line 983
def self.resolveYAMLAnchors(lines)
  new_text = ""
  lines.each_line { |line|
    if line.match(/# MU::Config\.include PLACEHOLDER /)
      $yaml_refs.each_pair { |anchor, data|
        if line.sub!(/^(\s+).*?# MU::Config\.include PLACEHOLDER #{Regexp.quote(anchor)} REDLOHECALP/, "")
          indent = $1
          MU::Config.resolveYAMLAnchors(data).each_line { |addline|
            line = line + indent + addline
          }
          break
        end
      }
    end
    new_text = new_text + line
  }
  return new_text
end
schema() click to toggle source

Accessor for our Basket of Kittens schema definition

# File modules/mu/config/schema_helpers.rb, line 48
def self.schema
  @@schema
end
schemaMerge(orig, new, cloud) click to toggle source

Deep merge a configuration hash so we can meld different cloud providers' schemas together, while preserving documentation differences

# File modules/mu/config/schema_helpers.rb, line 54
    def self.schemaMerge(orig, new, cloud)
      if new.is_a?(Hash)
        new.each_pair { |k, v|
          if cloud and k == "description" and v.is_a?(String) and !v.match(/\b#{Regexp.quote(cloud.upcase)}\b/) and !v.empty?
            new[k] = "+"+cloud.upcase+"+: "+v
          end
          if orig and orig.has_key?(k)
          elsif orig
            orig[k] = new[k]
          else
            orig = new
          end
          schemaMerge(orig[k], new[k], cloud)
        }
      elsif orig.is_a?(Array) and new
        orig.concat(new)
        orig.uniq!
      elsif new.is_a?(String)
        orig ||= ""
        orig += "\n" if !orig.empty? 
        orig += "+#{cloud.upcase}+: "+new
      else
# XXX I think this is a NOOP?
      end
    end
stripConfig(config) click to toggle source

Make a deep copy of a config hash and pare it down to only primitive types, even at the leaves. @param config [Hash] @return [Hash]

# File modules/mu/config.rb, line 103
def self.stripConfig(config)
  MU::Config.manxify(Marshal.load(Marshal.dump(MU.structToHash(config.dup))), remove_runtime_keys: true)
end
tags_primitive() click to toggle source

Configuration chunk for creating resource tags as an array of key/value pairs. @return [Hash]

# File modules/mu/config/schema_helpers.rb, line 139
def self.tags_primitive
  {
    "type" => "array",
    "minItems" => 1,
    "items" => {
      "description" => "Tags to apply to this resource. Will apply at the cloud provider level and in node groomers, where applicable.",
      "type" => "object",
      "title" => "tags",
      "required" => ["key", "value"],
      "additionalProperties" => false,
      "properties" => {
        "key" => {
          "type" => "string",
        },
        "value" => {
          "type" => "string",
        }
      }
    }
  }
end
tails() click to toggle source

Accessor for tails in our Basket of Kittens. This should be a superset of user-supplied parameters. It also has machine-generated parameterized behaviors.

# File modules/mu/config.rb, line 67
def self.tails
  @@tails
end

Public Instance Methods

adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil, credentials: nil, rules_only: false) click to toggle source

Generate configuration for the general-purpose admin firewall rulesets (security groups in AWS). Note that these are unique to regions and individual VPCs (as well as Classic, which is just a degenerate case of a VPC for our purposes. @param vpc [Hash]: A VPC reference as defined in our config schema. This originates with the calling resource, so we'll peel out just what we need (a name or cloud id of a VPC). @param admin_ip [String]: Optional string of an extra IP address to allow blanket access to the calling resource. @param cloud [String]: The parent resource's cloud plugin identifier @param region [String]: Cloud provider region, if applicable. @return [Hash<String>]: A dependency description that the calling resource can then add to itself.

# File modules/mu/config/firewall_rule.rb, line 144
def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil, credentials: nil, rules_only: false)
  if !cloud or (cloud == "AWS" and !region)
    raise MuError, "Cannot call adminFirewallRuleset without specifying the parent's region and cloud provider"
  end
  hosts = Array.new
  hosts << "#{MU.my_public_ip}/32" if MU.my_public_ip
  hosts << "#{MU.my_private_ip}/32" if MU.my_private_ip
  hosts << "#{MU.mu_public_ip}/32" if MU.mu_public_ip
  hosts << "#{admin_ip}/32" if admin_ip
  hosts.uniq!

  rules = []
  if cloud == "Google"
    rules = [
      { "ingress" => true, "proto" => "all", "hosts" => hosts },
      { "egress" => true, "proto" => "all", "hosts" => hosts }
    ]
  else
    rules = [
      { "proto" => "tcp", "port_range" => "0-65535", "hosts" => hosts },
      { "proto" => "udp", "port_range" => "0-65535", "hosts" => hosts },
      { "proto" => "icmp", "port_range" => "-1", "hosts" => hosts }
    ]
  end

  if rules_only
    return rules
  end

  name = "admin"
  name += credentials.to_s if credentials
  realvpc = nil
  if vpc
    realvpc = {}
    ['vpc_name', 'vpc_id'].each { |p|
      if vpc[p]
        vpc[p.sub(/^vpc_/, '')] = vpc[p] 
        vpc.delete(p)
      end
    }
    ['cloud', 'id', 'name', 'deploy_id', 'habitat', 'credentials'].each { |field|
      realvpc[field] = vpc[field] if !vpc[field].nil?
    }
    if !realvpc['id'].nil? and !realvpc['id'].empty?
      # Stupid kludge for Google cloud_ids which are sometimes URLs and
      # sometimes not. Requirements are inconsistent from scenario to
      # scenario.
      name = name + "-" + realvpc['id'].gsub(/.*\//, "")
      realvpc['id'] = getTail("id", value: realvpc['id'], prettyname: "Admin Firewall Ruleset #{name} Target VPC",  cloudtype: "AWS::EC2::VPC::Id") if realvpc["id"].is_a?(String)
    elsif !realvpc['name'].nil?
      name = name + "-" + realvpc['name']
    end
  end


  acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true, "credentials" => credentials }
  if cloud == "Google" and acl["vpc"] and acl["vpc"]["habitat"]
    acl['project'] = acl["vpc"]["habitat"]["id"] || acl["vpc"]["habitat"]["name"]
  end
  acl.delete("vpc") if !acl["vpc"]
  if !MU::Cloud.resourceClass(cloud, "FirewallRule").isGlobal? and !region.nil? and !region.empty?
    acl["region"] = region
  end
  @admin_firewall_rules << acl if !@admin_firewall_rules.include?(acl)
  return {"type" => "firewall_rule", "name" => name}
end
check_dependencies() click to toggle source

For our resources which specify intra-stack dependencies, make sure those dependencies are actually declared.

# File modules/mu/config.rb, line 882
def check_dependencies
  ok = true

  @config.each_pair { |type, values|
    next if !values.instance_of?(Array)
    _shortclass, cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(type, false)
    next if !cfg_name
    values.each { |resource|
      next if !resource.kind_of?(Hash) or resource["dependencies"].nil?
      addme = []
      deleteme = []

      resource["dependencies"].each { |dependency|
        dependency["their_phase"] ||= dependency["phase"]
        dependency.delete("phase")
        dependency["my_phase"] ||= dependency["no_create_wait"] ? "groom" : "create"
        dependency.delete("no_create_wait")
        # make sure the thing we depend on really exists
        sibling = haveLitterMate?(dependency['name'], dependency['type'])
        if !sibling
          MU.log "Missing dependency: #{type}{#{resource['name']}} needs #{cfg_name}{#{dependency['name']}}", MU::ERR
          ok = false
          next
        end

        # Fudge dependency declarations to quash virtual_names that we know
        # are extraneous. Note that wee can't do all virtual names here; we
        # have no way to guess which of a collection of resources is the
        # real correct one.
        if sibling['virtual_name'] == dependency['name']
          real_resources = []
          found_exact = false
          resource["dependencies"].each { |dep_again|
            if dep_again['type'] == dependency['type'] and sibling['name'] == dep_again['name']
              dependency['name'] = sibling['name']
              found_exact = true
              break
            end
          }
          if !found_exact
            all_siblings = haveLitterMate?(dependency['name'], dependency['type'], has_multiple: true)
            if all_siblings.size > 0
              all_siblings.each { |s|
                newguy = dependency.clone
                newguy['name'] = s['name']
                addme << newguy
              }
              deleteme << dependency
              MU.log "Expanding dependency which maps to virtual resources to all matching real resources", MU::NOTICE, details: { sibling['virtual_name'] => addme }
              next
            end
          end
        end

        if dependency['their_phase'] == "groom"
          sibling['dependencies'].each { |sib_dep|
            next if sib_dep['type'] != cfg_name or sib_dep['their_phase'] != "groom"
            cousin = haveLitterMate?(sib_dep['name'], sib_dep['type'])
            if cousin and cousin['name'] == resource['name']
              MU.log "Circular dependency between #{type} #{resource['name']} <=> #{dependency['type']} #{dependency['name']}", MU::ERR, details: [ resource['name'] => dependency, sibling['name'] => sib_dep ]
              ok = false
            end
          }
        end

        # Check for a circular relationship that will lead to a deadlock
        # when creating resource. This only goes one layer deep, and does
        # not consider groom-phase deadlocks.
        if dependency['their_phase'] == "groom" or
           dependency['my_phase'] == "groom" or (
             !MU::Cloud.resourceClass(sibling['cloud'], type).deps_wait_on_my_creation and
             !MU::Cloud.resourceClass(resource['cloud'], type).waits_on_parent_completion
           )
          next
        end

        if sibling['dependencies']
          sibling['dependencies'].each { |sib_dep|
            next if sib_dep['type'] != cfg_name or sib_dep['my_phase'] == "groom"
            cousin = haveLitterMate?(sib_dep['name'], sib_dep['type'])
            if cousin and cousin['name'] == resource['name']
              MU.log "Circular dependency between #{type} #{resource['name']} <=> #{dependency['type']} #{dependency['name']}", MU::ERR, details: [ resource['name'] => dependency, sibling['name'] => sib_dep ]
              ok = false
            end
          }
        end
      }
      resource["dependencies"].reject! { |dep| deleteme.include?(dep) }
      resource["dependencies"].concat(addme)
      resource["dependencies"].uniq!

    }
  }

  ok
end
cloudCode(code, placeholder = "CLOUDCODEPLACEHOLDER") click to toggle source

Instead of resolving a parameter, leave a placeholder for a cloud-specific variable that will be generated at runtime. Canonical use case: referring to a CloudFormation variable by reference, like “AWS::StackName” or “SomeChildTemplate.OutputVariableName.” @param code [String]: A string consistent of code which will be understood by the Cloud layer, e.g. '“Ref” : “AWS::StackName”' (CloudFormation) @param placeholder [Object]: A placeholder value to use at the config parser stage, if the default string will not pass validation.

# File modules/mu/config.rb, line 156
def cloudCode(code, placeholder = "CLOUDCODEPLACEHOLDER")
  var_name = code.gsub(/[^a-z0-9]/i, "_")
  placeholder = code if placeholder.nil?
  getTail(var_name, value: placeholder, runtimecode: code)
  "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP"
end
divideNetwork(ip_block, subnets_desired, max_mask = 28) click to toggle source

Take an IP block and split it into a more-or-less arbitrary number of subnets. @param ip_block [String]: CIDR of the network to subdivide @param subnets_desired [Integer]: Number of subnets we want back @param max_mask [Integer]: The highest netmask we're allowed to use for a subnet (various by cloud provider) @return [MU::Config::Tail]: Resulting subnet tails, or nil if an error occurred.

# File modules/mu/config/vpc.rb, line 1080
def divideNetwork(ip_block, subnets_desired, max_mask = 28)
  cidr = NetAddr::IPv4Net.parse(ip_block.to_s)

  # Ugly but reliable method of landing on the right subnet size
  subnet_bits = cidr.netmask.prefix_len
  begin
    subnet_bits += 1
    if subnet_bits > max_mask
      MU.log "Can't subdivide #{cidr.to_s} into #{subnets_desired.to_s}", MU::ERR
      raise MuError, "Subnets smaller than /#{max_mask} not permitted"
    end
  end while cidr.subnet_count(subnet_bits) < subnets_desired

  if cidr.subnet_count(subnet_bits) > subnets_desired
    MU.log "Requested #{subnets_desired.to_s} subnets from #{cidr.to_s}, leaving #{(cidr.subnet_count(subnet_bits)-subnets_desired).to_s} unused /#{subnet_bits.to_s}s available", MU::NOTICE
  end

  begin
    subnets = []
    (0..subnets_desired).each { |x|
      subnets << cidr.nth_subnet(subnet_bits, x).to_s
    }
  rescue RuntimeError => e
    if e.message.match(/exceeds subnets available for allocation/)
      MU.log e.message, MU::ERR
      MU.log "I'm attempting to create #{subnets_desired} subnets (one public and one private for each Availability Zone), of #{subnet_size} addresses each, but that's too many for a /#{cidr.netmask.prefix_len} network. Either declare a larger network, or explicitly declare a list of subnets with few enough entries to fit.", MU::ERR
      return nil
    else
      raise e
    end
  end

  subnets = getTail("subnetblocks", value: subnets.join(","), cloudtype: "CommaDelimitedList", description: "IP Address ranges to be used for VPC subnets", prettyname: "SubnetIpBlocks", list_of: "ip_block").map { |tail| tail["ip_block"] }
  subnets
end
getTail(param, value: nil, prettyname: nil, cloudtype: "String", valid_values: [], description: nil, list_of: nil, prefix: "", suffix: "", pseudo: false, runtimecode: nil) click to toggle source

Wrapper method for creating a {MU::Config::Tail} object as a reference to a parameter that's valid in the loaded configuration. @param param [<String>]: The name of the parameter to which this should be tied. @param value [<String>]: The value of the parameter to return when asked @param prettyname [<String>]: A human-friendly parameter name to be used when generating CloudFormation templates and the like @param cloudtype [<String>]: A platform-specific identifier used by cloud layers to identify a parameter's type, e.g. AWS::EC2::VPC::Id @param valid_values [Array<String>]: A list of acceptable String values for the given parameter. @param description [<String>]: A long-form description of what the parameter does. @param list_of [<String>]: Indicates that the value should be treated as a member of a list (array) by the cloud layer. @param prefix [<String>]: A static String that should be prefixed to the stored value when queried @param suffix [<String>]: A static String that should be appended to the stored value when queried @param pseudo [<Boolean>]: This is a pseudo-parameter, automatically provided, and not available as user input. @param runtimecode [<String>]: Actual code to allow the cloud layer to interpret literally in its own idiom, e.g. '“Ref” : “AWS::StackName”' for CloudFormation

# File modules/mu/config/tail.rb, line 145
    def getTail(param, value: nil, prettyname: nil, cloudtype: "String", valid_values: [], description: nil, list_of: nil, prefix: "", suffix: "", pseudo: false, runtimecode: nil)
      param = param.gsub(/[^a-z0-9_]/i, "_")
      if value.nil?
        if @@parameters.nil? or !@@parameters.has_key?(param)
          MU.log "Parameter '#{param}' (#{param.class.name}) referenced in config but not provided (#{caller[0]})", MU::DEBUG, details: @@parameters
          return nil
#          raise DeployParamError
        else
          value = @@parameters[param]
        end
      end
      if !prettyname.nil?
        prettyname.gsub!(/[^a-z0-9]/i, "") # comply with CloudFormation restrictions
      end
      if value.is_a?(MU::Config::Tail)
        MU.log "Parameter #{param} is using a nested parameter as a value. This rarely works, depending on the target cloud. YMMV.", MU::WARN
        tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode)
      elsif !list_of.nil? or (@@tails.has_key?(param) and @@tails[param].is_a?(Array))
        tail = []
        count = 0
        value.split(/\s*,\s*/).each { |subval|
          if @@tails.has_key?(param) and !@@tails[param][count].nil?
            subval = @@tails[param][count].values.first.to_s if subval.nil?
            list_of = @@tails[param][count].values.first.getName if list_of.nil?
            prettyname = @@tails[param][count].values.first.getPrettyName if prettyname.nil?
            description = @@tails[param][count].values.first.description if description.nil?
            valid_values = @@tails[param][count].values.first.valid_values if valid_values.nil? or valid_values.empty?
            cloudtype = @@tails[param][count].values.first.getCloudType if @@tails[param][count].values.first.getCloudType != "String"
          end
          prettyname = param.capitalize if prettyname.nil?
          tail << { list_of => MU::Config::Tail.new(list_of, subval, prettyname, cloudtype, valid_values, description, true, pseudo: pseudo, index: count) }
          count = count + 1
        }
      else
        if @@tails.has_key?(param)
          pseudo = @@tails[param].pseudo
          value = @@tails[param].to_s if value.nil?
          prettyname = @@tails[param].getPrettyName if prettyname.nil?
          description = @@tails[param].description if description.nil?
          valid_values = @@tails[param].valid_values if valid_values.nil? or valid_values.empty?
          cloudtype = @@tails[param].getCloudType if @@tails[param].getCloudType != "String"
        end
        tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode)
      end

      if valid_values and valid_values.size > 0 and value
        if !valid_values.include?(value)
          raise DeployParamError, "Invalid parameter value '#{value}' supplied for '#{param}'"
        end
      end
      @@tails[param] = tail

      tail
    end
haveLitterMate?(name, type, has_multiple: false) click to toggle source

See if a given resource is configured in the current stack @param name [String]: The name of the resource being checked @param type [String]: The type of resource being checked @return [Boolean]

# File modules/mu/config.rb, line 470
def haveLitterMate?(name, type, has_multiple: false)
  @kittencfg_semaphore.synchronize {
    matches = []
    _shortclass, _cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type)
    if @kittens[cfg_plural]
      @kittens[cfg_plural].each { |kitten|
        if kitten['name'].to_s == name.to_s or
           kitten['virtual_name'].to_s == name.to_s or
           (has_multiple and name.nil?)
          if has_multiple
            matches << kitten
          else
            return kitten
          end
        end
      }
    end
    if has_multiple
      return matches
    else
      return false
    end
  }
end
insertKitten(descriptor, type, delay_validation = false, ignore_duplicates: false, overwrite: false) click to toggle source

Insert a resource into the current stack @param descriptor [Hash]: The configuration description, as from a Basket of Kittens @param type [String]: The type of resource being added @param delay_validation [Boolean]: Whether to hold off on calling the resource's validateConfig method @param ignore_duplicates [Boolean]: Do not raise an exception if we attempt to insert a resource with a name field that's already in use

# File modules/mu/config.rb, line 519
    def insertKitten(descriptor, type, delay_validation = false, ignore_duplicates: false, overwrite: false)
      append = false
      start = Time.now

      shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
      MU.log "insertKitten on #{cfg_name} #{descriptor['name']} (delay_validation: #{delay_validation.to_s})", MU::DEBUG, details: caller[0]

      if overwrite
        removeKitten(descriptor['name'], type)
      end

      if !ignore_duplicates and haveLitterMate?(descriptor['name'], cfg_name)
#        raise DuplicateNameError, "A #{shortclass} named #{descriptor['name']} has already been inserted into this configuration"
      end

      @kittencfg_semaphore.synchronize {
        append = !@kittens[cfg_plural].include?(descriptor)

        # Skip if this kitten has already been validated and appended
        if !append and descriptor["#MU_VALIDATED"]
          return true
        end
      }
      ok = true

      if descriptor['cloud'] and
         !MU::Cloud.availableClouds.include?(descriptor['cloud'])
        if MU::Cloud.supportedClouds.include?(descriptor['cloud'])
          MU.log "#{cfg_name} #{descriptor['name']} is configured with cloud #{descriptor['cloud']}, but no #{descriptor['cloud']} credentials available", MU::ERR
        else
          MU.log "#{cfg_name} #{descriptor['name']}: Cloud provider #{descriptor['cloud']} is not supported", MU::ERR, details: MU::Cloud.supportedClouds
        end
        return false
      end

      descriptor["#MU_CLOUDCLASS"] = classname

      applyInheritedDefaults(descriptor, cfg_plural)

      # Meld defaults from our global schema and, if applicable, from our
      # cloud-specific schema.
      schemaclass = Object.const_get("MU").const_get("Config").const_get(shortclass)
      myschema = Marshal.load(Marshal.dump(MU::Config.schema["properties"][cfg_plural]["items"]))
      more_required, more_schema = MU::Cloud.resourceClass(descriptor["cloud"], type).schema(self)
      if more_schema
        MU::Config.schemaMerge(myschema["properties"], more_schema, descriptor["cloud"])
      end
      myschema["required"] ||= []
      if more_required
        myschema["required"].concat(more_required)
        myschema["required"].uniq!
      end

      descriptor = applySchemaDefaults(descriptor, myschema, type: shortclass)
      MU.log "Schema check on #{descriptor['cloud']} #{cfg_name} #{descriptor['name']}", MU::DEBUG, details: myschema

      if (descriptor["region"] and descriptor["region"].empty?) or
         (descriptor['cloud'] == "Google" and ["firewall_rule", "vpc"].include?(cfg_name))
        descriptor.delete("region")
      end

      # Make sure a sensible region has been targeted, if applicable
      classobj = MU::Cloud.cloudClass(descriptor["cloud"])
      if descriptor["region"]
        valid_regions = classobj.listRegions
        if !valid_regions.include?(descriptor["region"])
          MU.log "Known regions for cloud '#{descriptor['cloud']}' do not include '#{descriptor["region"]}'", MU::ERR, details: valid_regions
          ok = false
        end
      end

      if descriptor.has_key?('project')
        if descriptor['project'].nil?
          descriptor.delete('project')
        elsif haveLitterMate?(descriptor['project'], "habitats")
          MU::Config.addDependency(descriptor, descriptor['project'], "habitat")
        end
      end

      # Does this resource go in a VPC?
      if !descriptor["vpc"].nil? and !delay_validation
        # Quietly fix old vpc reference style
        if descriptor['vpc']['vpc_id']
          descriptor['vpc']['id'] ||= descriptor['vpc']['vpc_id']
          descriptor['vpc'].delete('vpc_id')
        end
        if descriptor['vpc']['vpc_name']
          descriptor['vpc']['name'] = descriptor['vpc']['vpc_name']
          descriptor['vpc'].delete('vpc_name')
        end

        descriptor['vpc']['cloud'] = descriptor['cloud']
        if descriptor['credentials']
          descriptor['vpc']['credentials'] ||= descriptor['credentials']
        end
        if descriptor['vpc']['region'].nil? and !descriptor['region'].nil? and !descriptor['region'].empty? and descriptor['vpc']['cloud'] != "Google"
          descriptor['vpc']['region'] = descriptor['region']
        end

        # If we're using a VPC in this deploy, set it as a dependency
        if !descriptor["vpc"]["name"].nil? and
           haveLitterMate?(descriptor["vpc"]["name"], "vpcs") and
           descriptor["vpc"]['deploy_id'].nil? and
           descriptor["vpc"]['id'].nil? and
           !(cfg_name == "vpc" and descriptor['name'] == descriptor['vpc']['name'])
          MU::Config.addDependency(descriptor, descriptor['vpc']['name'], "vpc")
          siblingvpc = haveLitterMate?(descriptor["vpc"]["name"], "vpcs")

          if siblingvpc and siblingvpc['bastion'] and
             ["server", "server_pool", "container_cluster"].include?(cfg_name) and
             !descriptor['bastion']
            if descriptor['name'] != siblingvpc['bastion']['name']
              MU::Config.addDependency(descriptor, siblingvpc['bastion']['name'], "server")
            end
          end

          # things that live in subnets need their VPCs to be fully
          # resolved before we can proceed
          if ["server", "server_pool", "loadbalancer", "database", "cache_cluster", "container_cluster", "storage_pool"].include?(cfg_name)
            if !siblingvpc["#MU_VALIDATED"]
              ok = false if !insertKitten(siblingvpc, "vpcs", overwrite: overwrite)
            end
          end
          if !MU::Config::VPC.processReference(descriptor['vpc'],
                                  cfg_plural,
                                  descriptor,
                                  self,
                                  dflt_region: descriptor['region'],
                                  credentials: descriptor['credentials'],
                                  dflt_project: descriptor['project'],
                                  sibling_vpcs: @kittens['vpcs'])
            ok = false
          end

          # If we're using a VPC from somewhere else, make sure the flippin'
          # thing exists, and also fetch its id now so later search routines
          # don't have to work so hard.
        else
          if !MU::Config::VPC.processReference(descriptor["vpc"],
                                  cfg_plural,
                                  descriptor,
                                  self,
                                  credentials: descriptor['credentials'],
                                  dflt_project: descriptor['project'],
                                  dflt_region: descriptor['region'])
            ok = false
          end
        end

        # if we didn't specify credentials but can inherit some from our target
        # VPC, do so
        if descriptor["vpc"]["credentials"]
          descriptor["credentials"] ||= descriptor["vpc"]["credentials"] 
        end

        # Clean crud out of auto-created VPC declarations so they don't trip
        # the schema validator when it's invoked later.
        if !["server", "server_pool", "database"].include?(cfg_name)
          descriptor['vpc'].delete("nat_ssh_user")
        end
        if descriptor['vpc']['cloud'] == "Google"
          descriptor['vpc'].delete("region")
        end
        if ["firewall_rule", "function"].include?(cfg_name)
          descriptor['vpc'].delete("subnet_pref")
        end
      end

      # Does it have generic ingress rules?
      fwname = cfg_name+descriptor['name']

      if (descriptor['ingress_rules'] or
         ["server", "server_pool", "database", "cache_cluster"].include?(cfg_name))
        descriptor['ingress_rules'] ||= []

        acl = haveLitterMate?(fwname, "firewall_rules")
        already_exists = !acl.nil?

        acl ||= {
          "name" => fwname,
          "rules" => descriptor['ingress_rules'],
          "region" => descriptor['region'],
          "credentials" => descriptor["credentials"]
        }
        if !MU::Cloud.resourceClass(descriptor["cloud"], "FirewallRule").isGlobal?
          acl['region'] = descriptor['region']
          acl['region'] ||= classobj.myRegion(acl['credentials'])
        else
          acl.delete("region")
        end
        if descriptor["vpc"]
          acl["vpc"] = descriptor['vpc'].dup
          acl["vpc"].delete("subnet_pref")
        end

        ["optional_tags", "tags", "cloud", "project"].each { |param|
          acl[param] = descriptor[param] if descriptor[param]
        }
        descriptor["add_firewall_rules"] ||= []
        descriptor["add_firewall_rules"] << {"name" => fwname, "type" => "firewall_rules" } # XXX why the duck is there a type argument required here?
        descriptor["add_firewall_rules"].uniq!

        acl = resolveIntraStackFirewallRefs(acl, delay_validation)
        ok = false if !insertKitten(acl, "firewall_rules", delay_validation, overwrite: already_exists)
      end

      # Does it declare association with any sibling LoadBalancers?
      if !descriptor["loadbalancers"].nil?
        descriptor["loadbalancers"].each { |lb|
          if !lb["concurrent_load_balancer"].nil?
            MU::Config.addDependency(descriptor, lb["concurrent_load_balancer"], "loadbalancer")
          end
        }
      end

      # Does it want to know about Storage Pools?
      if !descriptor["storage_pools"].nil?
        descriptor["storage_pools"].each { |sp|
          if sp["name"]
            MU::Config.addDependency(descriptor, sp["name"], "storage_pool")
          end
        }
      end

      # Does it declare association with first-class firewall_rules?
      if !descriptor["add_firewall_rules"].nil?
        descriptor["add_firewall_rules"].each { |acl_include|
          next if !acl_include["name"] and !acl_include["rule_name"]
          acl_include["name"] ||= acl_include["rule_name"]
          if haveLitterMate?(acl_include["name"], "firewall_rules")
            MU::Config.addDependency(descriptor, acl_include["name"], "firewall_rule", my_phase: ((cfg_name == "vpc") ? "groom" : "create"))
          elsif acl_include["name"]
            MU.log shortclass.to_s+" #{descriptor['name']} depends on FirewallRule #{acl_include["name"]}, but no such rule declared.", MU::ERR
            ok = false
          end
        }
      end

      # Does it declare some alarms?
      if descriptor["alarms"] && !descriptor["alarms"].empty?
        descriptor["alarms"].each { |alarm|
          alarm["name"] = "#{cfg_name}-#{descriptor["name"]}-#{alarm["name"]}"
          alarm['dimensions'] ||= []
          alarm["namespace"] ||= descriptor['name']
          alarm["credentials"] = descriptor["credentials"]
          alarm["#TARGETCLASS"] = cfg_name
          alarm["#TARGETNAME"] = descriptor['name']
          alarm['cloud'] = descriptor['cloud']

          ok = false if !insertKitten(alarm, "alarms", true, overwrite: overwrite)
        }
        descriptor.delete("alarms")
      end

      # Does it want to meld another deployment's resources into its metadata?
      if !descriptor["existing_deploys"].nil? and
         !descriptor["existing_deploys"].empty?
        descriptor["existing_deploys"].each { |ext_deploy|
          if ext_deploy["cloud_type"].nil?
            MU.log "You must provide a cloud_type", MU::ERR
            ok = false
          end

          if ext_deploy["cloud_id"]
            found = MU::MommaCat.findStray(
              descriptor['cloud'],
              ext_deploy["cloud_type"],
              cloud_id: ext_deploy["cloud_id"],
              region: descriptor['region'],
              dummy_ok: false
            ).first

            if found.nil?
              MU.log "Couldn't find existing #{ext_deploy["cloud_type"]} resource #{ext_deploy["cloud_id"]}", MU::ERR
              ok = false
            end
          elsif ext_deploy["mu_name"] && ext_deploy["deploy_id"]
            found = MU::MommaCat.findStray(
              descriptor['cloud'],
              ext_deploy["cloud_type"],
              deploy_id: ext_deploy["deploy_id"],
              mu_name: ext_deploy["mu_name"],
              region: descriptor['region'],
              dummy_ok: false
            ).first

            if found.nil?
              MU.log "Couldn't find existing #{ext_deploy["cloud_type"]} resource - #{ext_deploy["mu_name"]} / #{ext_deploy["deploy_id"]}", MU::ERR
              ok = false
            end
          else
            MU.log "Trying to find existing deploy, but either the cloud_id is not valid or no mu_name and deploy_id where provided", MU::ERR
            ok = false
          end
        }
      end

      if !delay_validation
        # Call the generic validation for the resource type, first and foremost
        # XXX this might have to be at the top of this insertKitten instead of
        # here
        ok = false if !schemaclass.validate(descriptor, self)

        plain_cfg = MU::Config.stripConfig(descriptor)
        plain_cfg.delete("#MU_CLOUDCLASS")
        plain_cfg.delete("#MU_VALIDATION_ATTEMPTED")
        plain_cfg.delete("#TARGETCLASS")
        plain_cfg.delete("#TARGETNAME")
        plain_cfg.delete("parent_block") if cfg_plural == "vpcs"
        begin
          JSON::Validator.validate!(myschema, plain_cfg)
        rescue JSON::Schema::ValidationError
          pp plain_cfg
          # Use fully_validate to get the complete error list, save some time
          errors = JSON::Validator.fully_validate(myschema, plain_cfg)
          realerrors = []
          errors.each { |err|
            if !err.match(/The property '.+?' of type MU::Config::Tail did not match the following type:/)
              realerrors << err
            end
          }
          if realerrors.size > 0
            MU.log "Validation error on #{descriptor['cloud']} #{cfg_name} #{descriptor['name']} (insertKitten called from #{caller[1]} with delay_validation=#{delay_validation}) #{@@config_path}!\n"+realerrors.join("\n"), MU::ERR, details: descriptor
            raise ValidationError, "Validation error on #{descriptor['cloud']} #{cfg_name} #{descriptor['name']} #{@@config_path}!\n"+realerrors.join("\n")
          end
        end

        # Run the cloud class's deeper validation, unless we've already failed
        # on stuff that will cause spurious alarms further in
        if ok
          parser = MU::Cloud.resourceClass(descriptor['cloud'], type)
          original_descriptor = MU::Config.stripConfig(descriptor)
          passed = parser.validateConfig(descriptor, self)

          if !passed
            descriptor = original_descriptor
            ok = false
          end

          # Make sure we've been configured with the right credentials
          cloudbase = MU::Cloud.cloudClass(descriptor['cloud'])
          credcfg = cloudbase.credConfig(descriptor['credentials'])
          if !credcfg or credcfg.empty?
            raise ValidationError, "#{descriptor['cloud']} #{cfg_name} #{descriptor['name']} declares credential set #{descriptor['credentials']}, but no such credentials exist for that cloud provider"
          end

          descriptor['#MU_VALIDATED'] = true
        end
      end

      descriptor["dependencies"].uniq! if descriptor["dependencies"]

      @kittencfg_semaphore.synchronize {
        @kittens[cfg_plural] << descriptor if append
      }

      MU.log "insertKitten completed #{cfg_name} #{descriptor['name']} in #{sprintf("%.2fs", Time.now-start)}", MU::DEBUG

      ok
    end
method_missing(var_name) click to toggle source

Catch calls to missing variables in Basket of Kittens files when being parsed by ERB, and replace with placeholders for parameters. This method_missing is only defined innside {MU::Config.resolveConfig}

# File modules/mu/config.rb, line 120
def method_missing(var_name)
  if @param_pass
    "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP"
  else
    tail = getTail(var_name.to_s)

    if tail.is_a?(Array)
      if @param_pass
        return tail.map {|f| f.values.first.to_s }.join(",")
      else
        # Don't try to jam complex types into a string file format, just
        # sub them back in later from a placeholder.
        return "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP"
      end
    else
      if @param_pass
        tail.to_s
      else
        return "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP"
      end
    end
  end
end
parameter?(var_name) click to toggle source

A check for the existence of a user-supplied parameter value that can be easily run in an ERB block in a Basket of Kittens.

# File modules/mu/config.rb, line 146
def parameter?(var_name)
  @@user_supplied_parameters.has_key?(var_name)
end
removeKitten(name, type) click to toggle source

Remove a resource from the current stack @param name [String]: The name of the resource being removed @param type [String]: The type of resource being removed

# File modules/mu/config.rb, line 498
def removeKitten(name, type)
  @kittencfg_semaphore.synchronize {
    _shortclass, _cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type)
    deletia = nil
    if @kittens[cfg_plural]
      @kittens[cfg_plural].each { |kitten|
        if kitten['name'] == name
          deletia = kitten
          break
        end
      }
      @kittens[type].delete(deletia) if !deletia.nil?
    end
  }
end
resolveConfig(path: @@config_path, param_pass: false, cloud: nil) click to toggle source

Load up our YAML or JSON and parse it through ERB, optionally substituting externally-supplied parameters.

# File modules/mu/config.rb, line 109
def resolveConfig(path: @@config_path, param_pass: false, cloud: nil)
  config = nil
  @param_pass = param_pass

  if cloud
    MU.log "Exposing cloud variable to ERB with value of #{cloud}", MU::DEBUG
  end

  # Catch calls to missing variables in Basket of Kittens files when being
  # parsed by ERB, and replace with placeholders for parameters. This
  # method_missing is only defined innside {MU::Config.resolveConfig}
  def method_missing(var_name)
    if @param_pass
      "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP"
    else
      tail = getTail(var_name.to_s)

      if tail.is_a?(Array)
        if @param_pass
          return tail.map {|f| f.values.first.to_s }.join(",")
        else
          # Don't try to jam complex types into a string file format, just
          # sub them back in later from a placeholder.
          return "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP"
        end
      else
        if @param_pass
          tail.to_s
        else
          return "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP"
        end
      end
    end
  end

  # A check for the existence of a user-supplied parameter value that can
  # be easily run in an ERB block in a Basket of Kittens.
  def parameter?(var_name)
    @@user_supplied_parameters.has_key?(var_name)
  end

  # Instead of resolving a parameter, leave a placeholder for a
  # cloud-specific variable that will be generated at runtime. Canonical
  # use case: referring to a CloudFormation variable by reference, like
  # "AWS::StackName" or "SomeChildTemplate.OutputVariableName."
  # @param code [String]: A string consistent of code which will be understood by the Cloud layer, e.g. '"Ref" : "AWS::StackName"' (CloudFormation)
  # @param placeholder [Object]: A placeholder value to use at the config parser stage, if the default string will not pass validation.
  def cloudCode(code, placeholder = "CLOUDCODEPLACEHOLDER")
    var_name = code.gsub(/[^a-z0-9]/i, "_")
    placeholder = code if placeholder.nil?
    getTail(var_name, value: placeholder, runtimecode: code)
    "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP"
  end

  # Make sure our parameter values are all available in the local namespace
  # that ERB will be using, minus any that conflict with existing variables
  erb_binding = get_binding(@@tails.keys.sort)
  @@tails.each_pair { |key, tail|
    next if !tail.is_a?(MU::Config::Tail) or tail.is_list_element
    # XXX figure out what to do with lists
    begin
      erb_binding.local_variable_set(key.to_sym, tail.to_s)
    rescue NameError
      MU.log "Binding #{key} = #{tail.to_s}", MU::DEBUG
      erb_binding.local_variable_set(key.to_sym, tail.to_s)
    end
  }

  # Figure out what kind of file we're loading. We handle includes
  # differently if YAML is involved. These globals get used inside
  # templates. They're globals on purpose. Stop whining.
  $file_format = MU::Config.guessFormat(path)
  $yaml_refs = {}
  erb = ERB.new(File.read(path), nil, "<>")
  erb.filename = path

  begin
    raw_text = erb.result(erb_binding)
  rescue NameError => e
    loc = e.backtrace[0].sub(/:(\d+):.*/, ':\1')
    msg = if e.message.match(/wrong constant name Config.getTail PLACEHOLDER ([^\s]+) REDLOHECALP/)
      "Variable '#{Regexp.last_match[1]}' referenced in config, but not defined. Missing required parameter?"
    else
      e.message
    end
    raise ValidationError, msg+" at "+loc
  end
  raw_json = nil

  # If we're working in YAML, do some magic to make includes work better.
  yaml_parse_error = nil
  if $file_format == :yaml
    begin
      raw_json = JSON.generate(YAML.load(MU::Config.resolveYAMLAnchors(raw_text)))
    rescue Psych::SyntaxError => e
      raw_json = raw_text
      yaml_parse_error = e.message
    end
  else
    raw_json = raw_text
  end

  begin
    config = JSON.parse(raw_json)
    if @@parameters['cloud']
      config['cloud'] ||= @@parameters['cloud'].to_s
    end
    if param_pass and config.is_a?(Hash)
      config.keys.each { |key|
        if key != "parameters"
          if key == "appname" and @@parameters["myAppName"].nil?
            $myAppName = config["appname"].upcase.dup
            $myAppName.freeze
            @@parameters["myAppName"] = getTail("myAppName", value: config["appname"].upcase, pseudo: true).to_s
          end
          config.delete(key)
        end
      }
    elsif config.is_a?(Hash)
      config.delete("parameters")
    end
  rescue JSON::ParserError => e
    badconf = File.new("/tmp/badconf.#{$$}", File::CREAT|File::TRUNC|File::RDWR, 0400)
    badconf.puts raw_text
    badconf.close
    if !yaml_parse_error.nil? and !path.match(/\.json/)
      MU.log "YAML Error parsing #{path}! Complete file dumped to /tmp/badconf.#{$$}", MU::ERR, details: yaml_parse_error
    else
      MU.log "JSON Error parsing #{path}! Complete file dumped to /tmp/badconf.#{$$}", MU::ERR, details: e.message
    end
    raise ValidationError
  end

  undef :method_missing
  return [MU::Config.fixDashes(config), raw_text]
end
resolveIntraStackFirewallRefs(acl, delay_validation = false) click to toggle source

FirewallRules can reference other FirewallRules, which means we need to do an extra pass to make sure we get all intra-stack dependencies correct. @param acl [Hash]: The configuration hash for the FirewallRule to check @return [Hash]

# File modules/mu/config/firewall_rule.rb, line 117
    def resolveIntraStackFirewallRefs(acl, delay_validation = false)
      acl["rules"].each { |acl_include|
        if acl_include['sgs']
          acl_include['sgs'].each { |sg_ref|
            if haveLitterMate?(sg_ref, "firewall_rules")
              MU::Config.addDependency(acl, sg_ref, "firewall_rule", my_phase: "groom")
              siblingfw = haveLitterMate?(sg_ref, "firewall_rules")
              if !siblingfw["#MU_VALIDATED"]
# XXX raise failure somehow
                insertKitten(siblingfw, "firewall_rules", delay_validation: delay_validation)
              end
            end
          }
        end
      }
      acl
    end
resolveTails(tree, indent= "") click to toggle source

Convert parameter entries that constitute whole config keys into {MU::Config::Tail} objects.

# File modules/mu/config.rb, line 374
def resolveTails(tree, indent= "")
  if tree.is_a?(Hash)
    tree.each_pair { |key, val|
      tree[key] = resolveTails(val, indent+" ")
    }
  elsif tree.is_a?(Array)
    newtree = []
    tree.each { |item|
      newtree << resolveTails(item, indent+" ")
    }
    tree = newtree
  elsif tree.is_a?(String) and tree.match(/^(.*?)MU::Config.getTail PLACEHOLDER (.+?) REDLOHECALP(.*)/)
    tree = getTail($2, prefix: $1, suffix: $3)
    if tree.nil? and @@tails.has_key?($2) # XXX why necessary?
      tree = @@tails[$2]
    end
  end
  return tree
end
visualizeDependencies() click to toggle source

Output the dependencies of this BoK stack as a directed acyclic graph. Very useful for debugging.

# File modules/mu/config/doc_helpers.rb, line 102
def visualizeDependencies
  # GraphViz won't like MU::Config::Tail, pare down to plain Strings
  config = MU::Config.stripConfig(@config)
  begin
    g = GraphViz.new(:G, :type => :digraph)
    # Generate a GraphViz node for each resource in this stack
    nodes = {}
    MU::Cloud.resource_types.each_pair { |classname, attrs|
      nodes[attrs[:cfg_name]] = {}
      if config.has_key?(attrs[:cfg_plural]) and config[attrs[:cfg_plural]]
        config[attrs[:cfg_plural]].each { |resource|
          nodes[attrs[:cfg_name]][resource['name']] = g.add_nodes("#{classname}: #{resource['name']}")
        }
      end
    }
    # Now add edges corresponding to the dependencies they list
    MU::Cloud.resource_types.values.each { |attrs|
      if config.has_key?(attrs[:cfg_plural]) and config[attrs[:cfg_plural]]
        config[attrs[:cfg_plural]].each { |resource|
          if resource.has_key?("dependencies")
            me = nodes[attrs[:cfg_name]][resource['name']]
            resource["dependencies"].each { |dep|
              parent = nodes[dep['type']][dep['name']]
              g.add_edges(me, parent)
            }
          end
        }
      end
    }
    # Spew some output?
    MU.log "Emitting dependency graph as /tmp/#{config['appname']}.jpg", MU::NOTICE
    g.output(:jpg => "/tmp/#{config['appname']}.jpg")
  rescue StandardError => e
    MU.log "Failed to generate GraphViz dependency tree: #{e.inspect}. This should only matter to developers.", MU::WARN, details: e.backtrace
  end
end

Private Instance Methods

applyInheritedDefaults(kitten, type) click to toggle source

Given a bare hash describing a resource, insert default values which can be inherited from its parent or from the root of the BoK. @param kitten [Hash]: A resource descriptor @param type [String]: The type of resource this is (“servers” etc)

# File modules/mu/config/schema_helpers.rb, line 338
    def applyInheritedDefaults(kitten, type)
      return if !kitten.is_a?(Hash)
      kitten['cloud'] ||= @config['cloud']
      kitten['cloud'] ||= MU::Config.defaultCloud

      if !MU::Cloud.supportedClouds.include?(kitten['cloud'])
        return
      end

      cloudclass = MU::Cloud.cloudClass(kitten['cloud'])

      resclass = MU::Cloud.resourceClass(kitten['cloud'], type)

      schema_fields = ["us_only", "scrub_mu_isms", "credentials", "billing_acct"]
      if !resclass.isGlobal?
        kitten['region'] ||= @config['region']
        kitten['region'] ||= cloudclass.myRegion(kitten['credentials'])
        schema_fields << "region"
      end

      kitten['credentials'] ||= @config['credentials']
      kitten['credentials'] ||= cloudclass.credConfig(name_only: true)

      kitten['us_only'] ||= @config['us_only']
      kitten['us_only'] ||= false

      kitten['scrub_mu_isms'] ||= @config['scrub_mu_isms']
      kitten['scrub_mu_isms'] ||= false

      if kitten['cloud'] == "Google"
# TODO this should be cloud-generic (handle AWS accounts, Azure subscriptions)
        if resclass.canLiveIn.include?(:Habitat)
          kitten["project"] ||= MU::Cloud::Google.defaultProject(kitten['credentials'])
          schema_fields << "project"
        end
        if kitten['region'].nil? and !kitten['#MU_CLOUDCLASS'].nil? and
           !resclass.isGlobal? and
           ![MU::Cloud::VPC, MU::Cloud::FirewallRule].include?(kitten['#MU_CLOUDCLASS'])
          if MU::Cloud::Google.myRegion((kitten['credentials'])).nil?
            raise ValidationError, "Google '#{type}' resource '#{kitten['name']}' declared without a region, but no default Google region declared in mu.yaml under #{kitten['credentials'].nil? ? "default" : kitten['credentials']} credential set" 
          end
          kitten['region'] ||= MU::Cloud::Google.myRegion
        end
      elsif kitten["cloud"] == "AWS" and !resclass.isGlobal? and !kitten['region']
        if MU::Cloud::AWS.myRegion.nil?
          raise ValidationError, "AWS resource declared without a region, but no default AWS region found"
        end
        kitten['region'] ||= MU::Cloud::AWS.myRegion
      end


      kitten['billing_acct'] ||= @config['billing_acct'] if @config['billing_acct']

      kitten["dependencies"] ||= []

      # Make sure the schema knows about these "new" fields, so that validation
      # doesn't trip over them.
      schema_fields.each { |field|
        if @@schema["properties"][field]
          MU.log "Adding #{field} to schema for #{type} #{kitten['cloud']}", MU::DEBUG, details: @@schema["properties"][field]
          @@schema["properties"][type]["items"]["properties"][field] ||= @@schema["properties"][field]
        end
      }
    end
applySchemaDefaults(conf_chunk = config, schema_chunk = schema, depth = 0, siblings = nil, type: nil) click to toggle source
# File modules/mu/config/schema_helpers.rb, line 276
def applySchemaDefaults(conf_chunk = config, schema_chunk = schema, depth = 0, siblings = nil, type: nil)
  return if schema_chunk.nil?

  if conf_chunk != nil and schema_chunk["properties"].kind_of?(Hash) and conf_chunk.is_a?(Hash)

    if schema_chunk["properties"]["creation_style"].nil? or
        schema_chunk["properties"]["creation_style"] != "existing"
      schema_chunk["properties"].each_pair { |key, subschema|
        shortclass = if conf_chunk[key]
          shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(key, false)
          shortclass
        else
          nil
        end

        new_val = applySchemaDefaults(conf_chunk[key], subschema, depth+1, conf_chunk, type: shortclass).dup
        if !new_val.nil?
          begin
            conf_chunk[key] = Marshal.load(Marshal.dump(new_val))
          rescue TypeError
            conf_chunk[key] = new_val.clone
          end
        end
      }
    end
  elsif schema_chunk["type"] == "array" and conf_chunk.kind_of?(Array)
    conf_chunk.map! { |item|
      # If we're working on a resource type, go get implementation-specific
      # schema information so that we set those defaults correctly.
      realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"] and MU::Cloud.supportedClouds.include?(item['cloud'])

        _toplevel_required, cloudschema = MU::Cloud.resourceClass(item["cloud"], type).schema(self)

        newschema = schema_chunk["items"].dup
        MU::Config.schemaMerge(newschema["properties"], cloudschema, item["cloud"])
        newschema
      else
        schema_chunk["items"].dup
      end

      applySchemaDefaults(item, realschema, depth+1, conf_chunk, type: type).dup
    }
  else
    if conf_chunk.nil? and !schema_chunk["default_if"].nil? and !siblings.nil?
      schema_chunk["default_if"].each { |cond|
        if siblings[cond["key_is"]] == cond["value_is"]
          return Marshal.load(Marshal.dump(cond["set"]))
        end
      }
    end
    if conf_chunk.nil? and schema_chunk["default"] != nil
      return Marshal.load(Marshal.dump(schema_chunk["default"]))
    end
  end

  return conf_chunk
end
get_binding(keyset) click to toggle source

Namespace magic to pass to ERB's result method.

# File modules/mu/config.rb, line 1159
    def get_binding(keyset)
      environment = $environment
      myPublicIp = $myPublicIp
      myRoot = $myRoot
      myAZ = $myAZ
      myRegion = $myRegion
      myAppName = $myAppName

#      return MU::Config.global_bindings[keyset] if MU::Config.global_bindings[keyset]
      MU::Config.global_bindings[keyset] = binding
      MU::Config.global_bindings[keyset]
    end
include(file) click to toggle source

(see include)

# File modules/mu/config.rb, line 1154
def include(file)
  MU::Config.include(file, get_binding(@@tails.keys.sort), @param_pass)
end
validate(config = @config) click to toggle source
# File modules/mu/config.rb, line 1172
    def validate(config = @config)
      ok = true

      count = 0
      @kittens ||= {}
      types = MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }

      types.each { |type|
        @kittens[type] = config[type]
        @kittens[type] ||= []
        @kittens[type].each { |k|
          applyInheritedDefaults(k, type)
        }
        count = count + @kittens[type].size
      }


      if count == 0
        MU.log "You must declare at least one resource to create", MU::ERR
        ok = false
      end

      @nat_routes ||= {}
      types.each { |type|
        @kittens[type].each { |descriptor|
          ok = false if !insertKitten(descriptor, type)
        }
      }

      newrules = []
      @kittens["firewall_rules"].each { |acl|
        newrules << resolveIntraStackFirewallRefs(acl)
      }
      @kittens["firewall_rules"] = newrules

      # VPCs do complex things in their cloud-layer validation that other
      # resources tend to need, like subnet allocation, so hit them early.
      @kittens["vpcs"].each { |vpc|
        ok = false if !insertKitten(vpc, "vpcs")
      }

      # Make sure validation has been called for all on-the-fly generated
      # resources.
      validated_something_new = false
      begin
        validated_something_new = false
        types.each { |type|
          @kittens[type].each { |descriptor|
            if !descriptor["#MU_VALIDATION_ATTEMPTED"]
              validated_something_new = true
              ok = false if !insertKitten(descriptor, type)
              descriptor["#MU_VALIDATION_ATTEMPTED"] = true
            end
          }
        }
      end while validated_something_new

      # Do another pass of resolving intra-stack VPC peering, in case an
      # early-parsing VPC needs more details from a later-parsing one
      @kittens["vpcs"].each { |vpc|
        ok = false if !MU::Config::VPC.resolvePeers(vpc, self)
      }

      # add some default holes to allow dependent instances into databases
      @kittens["databases"].each { |db|
        if db['port'].nil?
          db['port'] = 3306 if ["mysql", "aurora"].include?(db['engine'])
          db['port'] = 5432 if ["postgres"].include?(db['engine'])
          db['port'] = 1433 if db['engine'].match(/^sqlserver\-/)
          db['port'] = 1521 if db['engine'].match(/^oracle\-/)
        end

        ruleset = haveLitterMate?("database"+db['name'], "firewall_rules")
        if ruleset
          ["server_pools", "servers"].each { |type|
            _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type)
            @kittens[cfg_plural].each { |server|
              server["dependencies"].each { |dep|
                if dep["type"] == "database" and dep["name"] == db["name"]
                  # XXX this is AWS-specific, I think. We need to use source_tags to make this happen in Google. This logic probably needs to be dumped into the database layer.
                  ruleset["rules"] << {
                    "proto" => "tcp",
                    "port" => db["port"],
                    "sgs" => [cfg_name+server['name']]
                  }
                  MU::Config.addDependency(ruleset, cfg_name+server['name'], "firewall_rule", my_phase: "groom")
                end
              }
            }
          }
        end
      }

      seen = []
      # XXX seem to be not detecting duplicate admin firewall_rules in adminFirewallRuleset
      @admin_firewall_rules.each { |acl|
        next if seen.include?(acl['name'])
        ok = false if !insertKitten(acl, "firewall_rules")
        seen << acl['name']
      }
      types.each { |type|
        config[type] = @kittens[type] if @kittens[type].size > 0
      }
      ok = false if !check_dependencies

      # TODO enforce uniqueness of resource names
      raise ValidationError if !ok

# XXX Does commenting this out make sense? Do we want to apply it to top-level
# keys and ignore resources, which validate when insertKitten is called now?
#      begin
#        JSON::Validator.validate!(MU::Config.schema, plain_cfg)
#      rescue JSON::Schema::ValidationError => e
#        # Use fully_validate to get the complete error list, save some time
#        errors = JSON::Validator.fully_validate(MU::Config.schema, plain_cfg)
#        realerrors = []
#        errors.each { |err|
#          if !err.match(/The property '.+?' of type MU::Config::Tail did not match the following type:/)
#            realerrors << err
#          end
#        }
#        if realerrors.size > 0
#          raise ValidationError, "Validation error in #{@@config_path}!\n"+realerrors.join("\n")
#        end
#      end
    end