class MU::Config
Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.
Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.
Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.
Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.
Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.
Constants
- CIDR_DESCRIPTION
- CIDR_PATTERN
- CIDR_PRIMITIVE
Attributes
The path to the most recently loaded configuration file
Public Class Methods
Insert a dependency into the config hash of a resource, with sensible error checking and de-duplication. @param resource [Hash] @param name [String] @param type [String] @param phase [String] @param no_create_wait [Boolean]
# File modules/mu/config.rb, line 440 def self.addDependency(resource, name, type, their_phase: "create", my_phase: nil) if ![nil, "create", "groom"].include?(their_phase) raise MuError, "Invalid their_phase '#{their_phase}' while adding dependency #{type} #{name} to #{resource['name']}" end resource['dependencies'] ||= [] _shortclass, cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(type) resource['dependencies'].each { |dep| if dep['type'] == cfg_name and dep['name'].to_s == name.to_s dep["their_phase"] = their_phase if their_phase dep["my_phase"] = my_phase if my_phase return end } newdep = { "type" => cfg_name, "name" => name.to_s } newdep["their_phase"] = their_phase if their_phase newdep["my_phase"] = my_phase if my_phase resource['dependencies'] << newdep end
Configuration chunk for choosing a cloud provider @return [Hash]
# File modules/mu/config/schema_helpers.rb, line 163 def self.cloud_primitive { "type" => "string", # "default" => MU::Config.defaultCloud, # applyInheritedDefaults does this better "enum" => MU::Cloud.supportedClouds } end
The path to the most recently loaded configuration file
# File modules/mu/config.rb, line 49 def self.config_path @@config_path end
Configuration chunk for choosing a set of cloud credentials @return [Hash]
# File modules/mu/config/schema_helpers.rb, line 118 def self.credentials_primitive { "type" => "string", "description" => "Specify a non-default set of credentials to use when authenticating to cloud provider APIs, as listed in `mu.yaml` under each provider's subsection. If " } end
The default cloud provider for new resources. Must exist in MU.supportedClouds return [String]
# File modules/mu/config/schema_helpers.rb, line 22 def self.defaultCloud configured = {} MU::Cloud.supportedClouds.each { |cloud| if $MU_CFG[cloud.downcase] and !$MU_CFG[cloud.downcase].empty? configured[cloud] = $MU_CFG[cloud.downcase].size configured[cloud] += 0.5 if MU::Cloud.cloudClass(cloud).hosted? # tiebreaker end } if configured.size > 0 return configured.keys.sort { |a, b| configured[b] <=> configured[a] }.first else MU::Cloud.supportedClouds.each { |cloud| return cloud if MU::Cloud.cloudClass(cloud).hosted? } return MU::Cloud.supportedClouds.first end end
The default grooming agent for new resources. Must exist in MU.supportedGroomers
.
# File modules/mu/config/schema_helpers.rb, line 43 def self.defaultGroomer MU.localOnly ? "Ansible" : "Chef" end
JSON-schema for resource dependencies @return [Hash]
# File modules/mu/config/schema_helpers.rb, line 174 def self.dependencies_primitive { "type" => "array", "items" => { "type" => "object", "description" => "Declare other objects which this resource requires. This resource will wait until the others are available to create itself.", "required" => ["name", "type"], "additionalProperties" => false, "properties" => { "name" => {"type" => "string"}, "type" => { "type" => "string", "enum" => MU::Cloud.resource_types.values.map { |v| v[:cfg_name] } }, "my_phase" => { "type" => "string", "description" => "Which part of our creation process should be waiting?", "enum" => ["create", "groom"] }, "their_phase" => { "type" => "string", "description" => "Which part of the creation process of the resource we depend on should we wait for before starting our own creation? Defaults are usually sensible, but sometimes you want, say, a Server to wait on another Server to be completely ready (through its groom phase) before starting up.", "enum" => ["create", "groom"] }, "phase" => { "type" => "string", "description" => "Alias for {their_phase}", "enum" => ["create", "groom"] }, "no_create_wait" => { "type" => "boolean", "description" => "DEPRECATED- setting +true+ is the same as setting {my_phase} to +groom+; setting to +false+ is the same as setting {my_phase} to +create+. If both +no_create_wait+ and {my_phase} are specified, {my_phase} takes precedence." } } } } end
Accessor for our Basket of Kittens schema definition, with various cloud-specific details merged so we can generate documentation for them.
# File modules/mu/config/doc_helpers.rb, line 22 def self.docSchema docschema = Marshal.load(Marshal.dump(@@schema)) only_children = {} MU::Cloud.resource_types.each_pair { |classname, attrs| MU::Cloud.supportedClouds.each { |cloud| begin require "mu/providers/#{cloud.downcase}/#{attrs[:cfg_name]}" rescue LoadError next end _required, res_schema = MU::Cloud.resourceClass(cloud, classname).schema(self) docschema["properties"][attrs[:cfg_plural]]["items"]["description"] ||= "" docschema["properties"][attrs[:cfg_plural]]["items"]["description"] += "\n#\n# `#{cloud}`: "+MU::Cloud.resourceClass(cloud, classname).quality res_schema.each { |key, cfg| if !docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key] only_children[attrs[:cfg_plural]] ||= {} only_children[attrs[:cfg_plural]][key] ||= {} only_children[attrs[:cfg_plural]][key][cloud] = cfg end } } } # recursively chase down description fields in arrays and objects of our # schema and prepend stuff to them for documentation def self.prepend_descriptions(prefix, cfg) cfg["prefix"] = prefix if cfg["type"] == "array" and cfg["items"] cfg["items"] = prepend_descriptions(prefix, cfg["items"]) elsif cfg["type"] == "object" and cfg["properties"] cfg["properties"].keys.each { |key| cfg["properties"][key] = prepend_descriptions(prefix, cfg["properties"][key]) } end cfg end MU::Cloud.resource_types.each_pair { |classname, attrs| MU::Cloud.supportedClouds.each { |cloud| res_class = nil begin res_class = MU::Cloud.resourceClass(cloud, classname) rescue MU::Cloud::MuCloudResourceNotImplemented next end required, res_schema = res_class.schema(self) next if required.size == 0 and res_schema.size == 0 res_schema.each { |key, cfg| cfg["description"] ||= "" if !cfg["description"].empty? cfg["description"] = "\n# +"+cloud.upcase+"+: "+cfg["description"] end if docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key] schemaMerge(docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key], cfg, cloud) docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] ||= "" docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] += "\n"+(cfg["description"].match(/^#/) ? "" : "# ")+cfg["description"] MU.log "Munging #{cloud}-specific #{classname.to_s} schema into BasketofKittens => #{attrs[:cfg_plural]} => #{key}", MU::DEBUG, details: docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key] else if only_children[attrs[:cfg_plural]][key] prefix = only_children[attrs[:cfg_plural]][key].keys.map{ |x| x.upcase }.join(" & ")+" ONLY" cfg["description"].gsub!(/^\n#/, '') # so we don't leave the description blank in the "optional parameters" section cfg = prepend_descriptions(prefix, cfg) end docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key] = cfg end docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["clouds"] = {} docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["clouds"][cloud] = cfg } docschema['required'].concat(required) docschema['required'].uniq! } } docschema end
Generate a documentation-friendly dummy Ruby class for our mu.yaml main config.
# File modules/mu/config/doc_helpers.rb, line 141 def self.emitConfigAsRuby example = %Q{--- public_address: 1.2.3.4 mu_admin_email: egtlabs@eglobaltech.com mu_admin_name: Joe Schmoe mommacat_port: 2260 banner: My Example Mu Master mu_repository: git://github.com/cloudamatic/mu.git repos: - https://github.com/cloudamatic/mu_demo_platform allow_invade_foreign_vpcs: true ansible_dir: aws: egtdev: region: us-east-1 log_bucket_name: egt-mu-log-bucket default: true name: egtdev personal: region: us-east-2 log_bucket_name: my-mu-log-bucket name: personal google: egtlabs: project: egt-labs-admin credentials_file: /opt/mu/etc/google.json region: us-east4 log_bucket_name: hexabucket-761234 default: true } mu_yaml_schema = eval(%Q{ $NOOP = true load "#{MU.myRoot}/bin/mu-configure" $CONFIGURABLES }) return if mu_yaml_schema.nil? or !mu_yaml_schema.is_a?(Hash) muyamlpath = "#{MU.myRoot}/modules/mu/mu.yaml.rb" MU.log "Converting mu.yaml schema to Ruby objects in #{muyamlpath}" muyaml_rb = File.new(muyamlpath, File::CREAT|File::TRUNC|File::RDWR, 0644) muyaml_rb.puts "# Configuration schema for mu.yaml. See also {https://github.com/cloudamatic/mu/wiki/Configuration the Mu wiki}." muyaml_rb.puts "#" muyaml_rb.puts "# Example:" muyaml_rb.puts "#" muyaml_rb.puts "# <pre>" example.split(/\n/).each { |line| muyaml_rb.puts "# "+line+" " # markdooooown } muyaml_rb.puts "# </pre>" muyaml_rb.puts "module MuYAML" muyaml_rb.puts "\t# The configuration file format for Mu's main config file." MU::Config.printMuYamlSchema(muyaml_rb, [], { "subtree" => mu_yaml_schema }) muyaml_rb.puts "end" muyaml_rb.close end
Take the schema we've defined and create a dummy Ruby class tree out of it, basically so we can leverage Yard to document it.
# File modules/mu/config/doc_helpers.rb, line 198 def self.emitSchemaAsRuby kittenpath = "#{MU.myRoot}/modules/mu/kittens.rb" MU.log "Converting Basket of Kittens schema to Ruby objects in #{kittenpath}" kitten_rb = File.new(kittenpath, File::CREAT|File::TRUNC|File::RDWR, 0644) kitten_rb.puts "### THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT ###" kitten_rb.puts "#" kitten_rb.puts "#" kitten_rb.puts "#" kitten_rb.puts "module MU" kitten_rb.puts "class Config" kitten_rb.puts "\t# The configuration file format for Mu application stacks." self.printSchema(kitten_rb, ["BasketofKittens"], MU::Config.docSchema) kitten_rb.puts "end" kitten_rb.puts "end" kitten_rb.close end
We used to be inconsistent about config keys using dashes versus underscores. Now we've standardized on the latter. Be polite and translate for older configs, since we're not fussed about name collisions.
# File modules/mu/config.rb, line 1047 def self.fixDashes(conf) if conf.is_a?(Hash) newhash = Hash.new conf.each_pair { |key, val| if val.is_a?(Hash) or val.is_a?(Array) val = self.fixDashes(val) end if key.match(/-/) MU.log "Replacing #{key} with #{key.gsub(/-/, "_")}", MU::DEBUG newhash[key.gsub(/-/, "_")] = val else newhash[key] = val end } return newhash elsif conf.is_a?(Array) conf.map! { |val| if val.is_a?(Hash) or val.is_a?(Array) self.fixDashes(val) else val end } end return conf end
Keep a cache of bindings we've created as sandbox contexts for ERB processing, so we don't keep reloading the entire Mu library inside new ones.
# File modules/mu/config.rb, line 1147 def self.global_bindings @@bindings end
Given a path to a config file, try to guess whether it's YAML or JSON. @param path [String]: The path to the file to check.
# File modules/mu/config.rb, line 1004 def self.guessFormat(path) raw = File.read(path) # Rip out ERB references that will bollocks parser syntax, first. stripped = raw.gsub(/<%.*?%>,?/, "").gsub(/,[\n\s]*([\]\}])/, '\1') begin JSON.parse(stripped) rescue JSON::ParserError begin YAML.load(raw.gsub(/<%.*?%>/, "")) rescue Psych::SyntaxError # Ok, well neither of those worked, let's assume that filenames are # meaningful. if path.match(/\.(yaml|yml)$/i) MU.log "Guessing that #{path} is YAML based on filename", MU::DEBUG return :yaml elsif path.match(/\.(json|jsn|js)$/i) MU.log "Guessing that #{path} is JSON based on filename", MU::DEBUG return :json else # For real? Ok, let's try the dumbest possible method. dashes = raw.match(/\-/) braces = raw.match(/[{}]/) if dashes.size > braces.size MU.log "Guessing that #{path} is YAML by... counting dashes.", MU::NOTICE return :yaml elsif braces.size > dashes.size MU.log "Guessing that #{path} is JSON by... counting braces.", MU::NOTICE return :json else raise "Unable to guess composition of #{path} by any means" end end end MU.log "Guessing that #{path} is YAML based on parser", MU::DEBUG return :yaml end MU.log "Guessing that #{path} is JSON based on parser", MU::NOTICE return :json end
This can be called with ERB from within a stack config file, like so: <%= Config.include
(“drupal.json”) %> It will first try the literal path you pass it, and if it fails to find that it will look in the directory containing the main (top-level) config.
# File modules/mu/config.rb, line 1081 def self.include(file, binding = nil, param_pass = false) loglevel = param_pass ? MU::NOTICE : MU::DEBUG retries = 0 orig_filename = file assume_type = nil if file.match(/(js|json|jsn)$/i) assume_type = :json elsif file.match(/(yaml|yml)$/i) assume_type = :yaml end begin erb = ERB.new(File.read(file), nil, "<>") rescue Errno::ENOENT retries = retries + 1 if retries == 1 file = File.dirname(MU::Config.config_path)+"/"+orig_filename retry elsif retries == 2 file = File.dirname(MU.myRoot)+"/lib/demo/"+orig_filename retry else raise ValidationError, "Couldn't read #{file} included from #{MU::Config.config_path}" end end begin # Include as just a drop-in block of text if the filename doesn't imply # a particular format, or if we're melding JSON into JSON. if ($file_format == :json and assume_type == :json) or assume_type.nil? MU.log "Including #{file} as uninterpreted text", loglevel return erb.result(binding) end # ...otherwise, try to parse into something useful so we can meld # differing file formats, or work around YAML's annoying dependence # on indentation. parsed_cfg = nil begin parsed_cfg = JSON.parse(erb.result(binding)) # parsed_as = :json rescue JSON::ParserError => e MU.log e.inspect, MU::DEBUG begin parsed_cfg = YAML.load(MU::Config.resolveYAMLAnchors(erb.result(binding))) # parsed_as = :yaml rescue Psych::SyntaxError => e MU.log e.inspect, MU::DEBUG MU.log "#{file} parsed neither as JSON nor as YAML, including as raw text", MU::WARN if @param_pass return erb.result(binding) end end if $file_format == :json MU.log "Including #{file} as interpreted JSON", loglevel return JSON.generate(parsed_cfg) else MU.log "Including #{file} as interpreted YAML", loglevel $yaml_refs[file] = ""+YAML.dump(parsed_cfg).sub(/^---\n/, "") return "# MU::Config.include PLACEHOLDER #{file} REDLOHECALP" end rescue SyntaxError raise ValidationError, "ERB in #{file} threw a syntax error" end end
Load and validate the schema for an individual resource class, optionally merging cloud-specific schema components. @param type [String]: The resource type to load @param cloud [String]: A specific cloud, whose implementation's schema of this resource we will merge @return [Hash]
# File modules/mu/config/schema_helpers.rb, line 228 def self.loadResourceSchema(type, cloud: nil) valid = true shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(type) schemaclass = Object.const_get("MU").const_get("Config").const_get(shortclass) [:schema, :validate].each { |method| if !schemaclass.respond_to?(method) MU.log "MU::Config::#{type}.#{method.to_s} doesn't seem to be implemented", MU::ERR return [nil, false] if method == :schema valid = false end } schema = schemaclass.schema.dup schema["properties"]["virtual_name"] = { "description" => "Internal use.", "type" => "string" } schema["properties"]["dependencies"] = MU::Config.dependencies_primitive schema["properties"]["cloud"] = MU::Config.cloud_primitive schema["properties"]["credentials"] = MU::Config.credentials_primitive schema["title"] = type.to_s if cloud cloudclass = MU::Cloud.resourceClass(cloud, type) if cloudclass.respond_to?(:schema) _reqd, cloudschema = cloudclass.schema cloudschema.each { |key, cfg| if schema["properties"][key] schemaMerge(schema["properties"][key], cfg, cloud) else schema["properties"][key] = cfg.dup end } else MU.log "MU::Cloud::#{cloud}::#{type}.#{method.to_s} doesn't seem to be implemented", MU::ERR valid = false end end return [schema, valid] end
Run through a config hash and return a version with all {MU::Config::Tail} endpoints converted to plain strings. Useful for cloud layers that don't care about the metadata in Tails. @param config [Hash]: The configuration tree to convert @return [Hash]: The modified configuration
# File modules/mu/config.rb, line 76 def self.manxify(config, remove_runtime_keys: false) if config.is_a?(Hash) newhash = {} config.each_pair { |key, val| next if remove_runtime_keys and (key.nil? or key.match(/^#MU_/)) next if val.is_a?(Array) and val.empty? newhash[key] = self.manxify(val, remove_runtime_keys: remove_runtime_keys) } config = newhash elsif config.is_a?(Array) newarray = [] config.each { |val| newarray << self.manxify(val, remove_runtime_keys: remove_runtime_keys) } config = newarray elsif config.is_a?(MU::Config::Tail) return config.to_s elsif config.is_a?(MU::Config::Ref) return self.manxify(config.to_h, remove_runtime_keys: remove_runtime_keys) end return config end
Load, resolve, and validate a configuration file (“Basket of Kittens”). @param path [String]: The path to the master config file to load. Note that this can include other configuration files via ERB. @param skipinitialupdates [Boolean]: Whether to forcibly apply the skipinitialupdates flag to nodes created by this configuration. @param params [Hash]: Optional name-value parameter pairs, which will be passed to our configuration files as ERB variables. @param cloud [String]: Sets a parameter named 'cloud', and insert it as the default cloud platform if not already declared @return [Hash]: The complete validated configuration for a deployment.
# File modules/mu/config.rb, line 257 def initialize(path, skipinitialupdates = false, params: {}, updating: nil, default_credentials: nil, cloud: nil) $myPublicIp ||= MU.mu_public_ip $myRoot ||= MU.myRoot $myRoot.freeze $myAZ ||= MU.myAZ.freeze $myAZ.freeze $myRegion ||= MU.curRegion.freeze $myRegion.freeze @kittens = {} @kittencfg_semaphore = Mutex.new @@config_path = path @admin_firewall_rules = [] @skipinitialupdates = skipinitialupdates @updating = updating if @updating @existing_deploy = MU::MommaCat.new(@updating) end @default_credentials = default_credentials ok = true params.each_pair { |name, value| begin raise DeployParamError, "Parameter must be formatted as name=value" if value.nil? or value.empty? raise DeployParamError, "Parameter name must be a legal Ruby variable name" if name.match(/[^A-Za-z0-9_]/) raise DeployParamError, "Parameter values cannot contain quotes" if value.match(/["']/) eval("defined? $#{name} and raise DeployParamError, 'Parameter name reserved'") @@parameters[name] = value @@user_supplied_parameters[name] = value eval("$#{name}='#{value}'") # support old-style $global parameter refs MU.log "Passing variable $#{name} into #{@@config_path} with value '#{value}'" rescue RuntimeError, SyntaxError => e ok = false MU.log "Error setting $#{name}='#{value}': #{e.message}", MU::ERR end } if cloud and !@@parameters["cloud"] if !MU::Cloud.availableClouds.include?(cloud) ok = false MU.log "Provider '#{cloud}' is not listed as an available cloud", MU::ERR, details: MU::Cloud.availableClouds else @@parameters["cloud"] = getTail("cloud", value: cloud, pseudo: true) @@user_supplied_parameters["cloud"] = cloud eval("$cloud='#{cloud}'") # support old-style $global parameter refs end end raise ValidationError if !ok # Run our input through the ERB renderer, a first pass just to extract # the parameters section so that we can resolve all of those to variables # for the rest of the config to reference. # XXX Figure out how to make include() add parameters for us. Right now # you can't specify parameters in an included file, because ERB is what's # doing the including, and parameters need to already be resolved so that # ERB can use them. param_cfg, _raw_erb_params_only = resolveConfig(path: @@config_path, param_pass: true, cloud: cloud) if param_cfg.has_key?("parameters") param_cfg["parameters"].each { |param| if param.has_key?("default") and param["default"].nil? param["default"] = "" end } end # Set up special Tail objects for our automatic pseudo-parameters getTail("myPublicIp", value: $myPublicIp, pseudo: true) getTail("myRoot", value: $myRoot, pseudo: true) getTail("myAZ", value: $myAZ, pseudo: true) getTail("myRegion", value: $myRegion, pseudo: true) if param_cfg.has_key?("parameters") and !param_cfg["parameters"].nil? and param_cfg["parameters"].size > 0 param_cfg["parameters"].each { |param| param['valid_values'] ||= [] if !@@parameters.has_key?(param['name']) if param.has_key?("default") @@parameters[param['name']] = param['default'].nil? ? "" : param['default'] elsif param["required"] or !param.has_key?("required") MU.log "Required parameter '#{param['name']}' not supplied", MU::ERR ok = false next else # not required, no default next end end if param.has_key?("cloudtype") getTail(param['name'], value: @@parameters[param['name']], cloudtype: param["cloudtype"], valid_values: param['valid_values'], description: param['description'], prettyname: param['prettyname'], list_of: param['list_of']) else getTail(param['name'], value: @@parameters[param['name']], valid_values: param['valid_values'], description: param['description'], prettyname: param['prettyname'], list_of: param['list_of']) end } end raise ValidationError if !ok @@parameters.each_pair { |name, val| next if @@tails.has_key?(name) and @@tails[name].is_a?(MU::Config::Tail) and @@tails[name].pseudo # Parameters can have limited parameterization of their own if @@tails[name].to_s.match(/^(.*?)MU::Config.getTail PLACEHOLDER (.+?) REDLOHECALP(.*)/) @@tails[name] = getTail(name, value: @@tails[$2]) end if respond_to?(name.to_sym) MU.log "Parameter name '#{name}' reserved", MU::ERR ok = false next end MU.log "Passing variable '#{name}' into #{path} with value '#{val}'" } raise DeployParamError, "One or more invalid parameters specified" if !ok $parameters = @@parameters.dup $parameters.freeze tmp_cfg, _raw_erb = resolveConfig(path: @@config_path, cloud: cloud) # Convert parameter entries that constitute whole config keys into # {MU::Config::Tail} objects. def resolveTails(tree, indent= "") if tree.is_a?(Hash) tree.each_pair { |key, val| tree[key] = resolveTails(val, indent+" ") } elsif tree.is_a?(Array) newtree = [] tree.each { |item| newtree << resolveTails(item, indent+" ") } tree = newtree elsif tree.is_a?(String) and tree.match(/^(.*?)MU::Config.getTail PLACEHOLDER (.+?) REDLOHECALP(.*)/) tree = getTail($2, prefix: $1, suffix: $3) if tree.nil? and @@tails.has_key?($2) # XXX why necessary? tree = @@tails[$2] end end return tree end @config = resolveTails(tmp_cfg) @config.merge!(param_cfg) if !@config.has_key?('admins') or @config['admins'].size == 0 @config['admins'] = [ { "name" => MU.chef_user == "mu" ? "Mu Administrator" : MU.userName, "email" => MU.userEmail } ] end @config['credentials'] ||= @default_credentials if @config['cloud'] and !MU::Cloud.availableClouds.include?(@config['cloud']) if MU::Cloud.supportedClouds.include?(@config['cloud']) MU.log "Cloud provider #{@config['cloud']} declared, but no #{@config['cloud']} credentials available", MU::ERR else MU.log "Cloud provider #{@config['cloud']} is not supported", MU::ERR, details: MU::Cloud.supportedClouds end exit 1 end MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }.each { |type| if @config[type] @config[type].each { |k| next if !k.is_a?(Hash) applyInheritedDefaults(k, type) } end } applySchemaDefaults(@config, MU::Config.schema) validate # individual resources validate when added now, necessary because the schema can change depending on what cloud they're targeting # XXX but now we're not validating top-level keys, argh #pp @config #raise "DERP" @config.freeze end
Have a default value available for config schema elements that take an email address. @return [String]
# File modules/mu/config/schema_helpers.rb, line 215 def self.notification_email if MU.chef_user == "mu" ENV['MU_ADMIN_EMAIL'] else MU.userEmail end end
Accessor for parameters to our Basket of Kittens
# File modules/mu/config.rb, line 59 def self.parameters @@parameters end
recursively chase down description fields in arrays and objects of our schema and prepend stuff to them for documentation
# File modules/mu/config/doc_helpers.rb, line 47 def self.prepend_descriptions(prefix, cfg) cfg["prefix"] = prefix if cfg["type"] == "array" and cfg["items"] cfg["items"] = prepend_descriptions(prefix, cfg["items"]) elsif cfg["type"] == "object" and cfg["properties"] cfg["properties"].keys.each { |key| cfg["properties"][key] = prepend_descriptions(prefix, cfg["properties"][key]) } end cfg end
Emit our mu.yaml schema in a format that YARD can comprehend and turn into documentation.
# File modules/mu/config/doc_helpers.rb, line 390 def self.printMuYamlSchema(muyaml_rb, class_hierarchy, schema, in_array = false, required = false) return if schema.nil? if schema["subtree"] printme = Array.new # order sub-elements by whether they're required, so we can use YARD's # grouping tags on them have_required = schema["subtree"].keys.any? { |k| schema["subtree"][k]["required"] } prop_list = schema["subtree"].keys.sort { |a, b| if schema["subtree"][a]["required"] and !schema["subtree"][b]["required"] -1 elsif !schema["subtree"][a]["required"] and schema["subtree"][b]["required"] 1 else a <=> b end } req = false printme << "# @!group Optional parameters" if !have_required prop_list.each { |name| prop = schema["subtree"][name] if prop["required"] printme << "# @!group Required parameters" if !req req = true else if req printme << "# @!endgroup" printme << "# @!group Optional parameters" end req = false end printme << self.printMuYamlSchema(muyaml_rb, class_hierarchy+ [name], prop, false, req) } printme << "# @!endgroup" desc = (schema['desc'] || schema['title']) tabs = 1 class_hierarchy.each { |classname| if classname == class_hierarchy.last and desc muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{desc}\n" end muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}" tabs = tabs + 1 } printme.each { |lines| if !lines.nil? and lines.is_a?(String) lines.lines.each { |line| muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + line } end } # class_hierarchy.each { |classname| # tabs = tabs - 1 # muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end" # } i = class_hierarchy.size until i == 0 do tabs = tabs - 1 muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end" i -= 1 end # And now that we've dealt with our children, pass our own rendered # commentary back up to our caller. name = class_hierarchy.last if in_array type = "Array<#{class_hierarchy.join("::")}>" else type = class_hierarchy.join("::") end docstring = "\n" docstring = docstring + "# **REQUIRED**\n" if required # docstring = docstring + "# **"+schema["prefix"]+"**\n" if schema["prefix"] docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if desc docstring = docstring + "#\n" docstring = docstring + "# @return [#{type}]\n" docstring = docstring + "# @see #{class_hierarchy.join("::")}\n" docstring = docstring + "attr_accessor :#{name}" return docstring else in_array = schema["array"] name = class_hierarchy.last type = if schema['boolean'] "Boolean" else "String" end if in_array type = "Array<#{type}>" end docstring = "\n" prefixes = [] prefixes << "# **REQUIRED**" if schema["required"] and schema['default'].nil? # prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"] prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil? if !schema['pattern'].nil? # XXX unquoted regex chars confuse the hell out of YARD. How do we # quote {}[] etc in YARD-speak? prefixes << "# **Must match pattern `#{schema['pattern'].to_s.gsub(/\n/, "\n#")}`**" end desc = (schema['desc'] || schema['title']) if prefixes.size > 0 docstring += prefixes.join(",\n") if desc and desc.size > 1 docstring += " - " end docstring += "\n" end docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if !desc.nil? docstring = docstring + "#\n" docstring = docstring + "# @return [#{type}]\n" docstring = docstring + "attr_accessor :#{name}" return docstring end end
Emit our Basket of Kittens schema in a format that YARD can comprehend and turn into documentation.
# File modules/mu/config/doc_helpers.rb, line 218 def self.printSchema(kitten_rb, class_hierarchy, schema, in_array = false, required = false, prefix: nil) return if schema.nil? if schema["type"] == "object" printme = [] if !schema["properties"].nil? # order sub-elements by whether they're required, so we can use YARD's # grouping tags on them if !schema["required"].nil? and schema["required"].size > 0 prop_list = schema["properties"].keys.sort_by { |name| schema["required"].include?(name) ? 0 : 1 } else prop_list = schema["properties"].keys end req = false printme << "# @!group Optional parameters" if schema["required"].nil? or schema["required"].size == 0 prop_list.each { |name| prop = schema["properties"][name] if class_hierarchy.size == 1 _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(name, false) if cfg_name example_path = MU.myRoot+"/modules/mu/config/"+cfg_name+".yml" if File.exist?(example_path) example = "#\n# Examples:\n#\n" # XXX these variables are all parameters from the BoKs in # modules/tests. A really clever implementation would read # and parse them to get default values, perhaps, instead of # hard-coding them here. instance_type = "t2.medium" db_size = "db.t2.medium" vpc_name = "some_vpc" logs_name = "some_loggroup" queues_name = "some_queue" server_pools_name = "some_server_pool" ["simple", "complex"].each { |complexity| erb = ERB.new(File.read(example_path), nil, "<>") example += "# !!!yaml\n" example += "# ---\n" example += "# appname: #{complexity}\n" example += "# #{cfg_plural}:\n" firstline = true erb.result(binding).split(/\n/).each { |l| l.chomp! l.sub!(/#.*/, "") if !l.match(/#(?:INTERNET|NAT|DENY)/) next if l.empty? or l.match(/^\s+$/) if firstline l = "- "+l firstline = false else l = " "+l end example += "# "+l+" "+"\n" } example += "# \n#\n" if complexity == "simple" } schema["properties"][name]["items"]["description"] ||= "" if !schema["properties"][name]["items"]["description"].empty? schema["properties"][name]["items"]["description"] += "\n" end schema["properties"][name]["items"]["description"] += example end end end if !schema["required"].nil? and schema["required"].include?(name) printme << "# @!group Required parameters" if !req req = true else if req printme << "# @!endgroup" printme << "# @!group Optional parameters" end req = false end printme << self.printSchema(kitten_rb, class_hierarchy+ [name], prop, false, req, prefix: schema["prefix"]) } printme << "# @!endgroup" end tabs = 1 class_hierarchy.each { |classname| if classname == class_hierarchy.last and !schema['description'].nil? kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{schema['description']}\n" end kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}" tabs = tabs + 1 } printme.each { |lines| if !lines.nil? and lines.is_a?(String) lines.lines.each { |line| kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + line } end } i = class_hierarchy.size until i == 0 do tabs = tabs - 1 kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end" i -= 1 end # And now that we've dealt with our children, pass our own rendered # commentary back up to our caller. name = class_hierarchy.last if in_array type = "Array<#{class_hierarchy.join("::")}>" else type = class_hierarchy.join("::") end docstring = "\n" docstring = docstring + "# **REQUIRED**\n" if required docstring = docstring + "# **"+schema["prefix"]+"**\n" if schema["prefix"] docstring = docstring + "# #{schema['description'].gsub(/\n/, "\n#")}\n" if !schema['description'].nil? docstring = docstring + "#\n" docstring = docstring + "# @return [#{type}]\n" docstring = docstring + "# @see #{class_hierarchy.join("::")}\n" docstring = docstring + "attr_accessor :#{name}" return docstring elsif schema["type"] == "array" return self.printSchema(kitten_rb, class_hierarchy, schema['items'], true, required, prefix: prefix) else name = class_hierarchy.last if schema['type'].nil? MU.log "Couldn't determine schema type in #{class_hierarchy.join(" => ")}", MU::WARN, details: schema return nil end if in_array type = "Array<#{schema['type'].capitalize}>" else type = schema['type'].capitalize end docstring = "\n" prefixes = [] prefixes << "# **REQUIRED**" if required and schema['default'].nil? prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"] prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil? if !schema['enum'].nil? and !schema["enum"].empty? prefixes << "# **Must be one of: `#{schema['enum'].join(', ')}`**" elsif !schema['pattern'].nil? # XXX unquoted regex chars confuse the hell out of YARD. How do we # quote {}[] etc in YARD-speak? prefixes << "# **Must match pattern `#{schema['pattern'].gsub(/\n/, "\n#")}`**" end if prefixes.size > 0 docstring += prefixes.join(",\n") if schema['description'] and schema['description'].size > 1 docstring += " - " end docstring += "\n" end docstring = docstring + "# #{schema['description'].gsub(/\n/, "\n#")}\n" if !schema['description'].nil? docstring = docstring + "#\n" docstring = docstring + "# @return [#{type}]\n" docstring = docstring + "attr_accessor :#{name}" return docstring end end
Configuration chunk for choosing a provider region @return [Hash]
# File modules/mu/config/schema_helpers.rb, line 95 def self.region_primitive if !@@allregions or @@allregions.empty? @@allregions = [] MU::Cloud.availableClouds.each { |cloud| next if @@loadfails.include?(cloud) cloudclass = MU::Cloud.cloudClass(cloud) begin return @@allregions if !cloudclass.listRegions() @@allregions.concat(cloudclass.listRegions()) rescue MU::MuError => e @@loadfails << cloud MU.log e.message, MU::WARN end } end { "type" => "string", "enum" => @@allregions } end
Ugly text-manipulation to recursively resolve some placeholder strings we put in for ERB include() directives. @param lines [String] @return [String]
# File modules/mu/config.rb, line 983 def self.resolveYAMLAnchors(lines) new_text = "" lines.each_line { |line| if line.match(/# MU::Config\.include PLACEHOLDER /) $yaml_refs.each_pair { |anchor, data| if line.sub!(/^(\s+).*?# MU::Config\.include PLACEHOLDER #{Regexp.quote(anchor)} REDLOHECALP/, "") indent = $1 MU::Config.resolveYAMLAnchors(data).each_line { |addline| line = line + indent + addline } break end } end new_text = new_text + line } return new_text end
Accessor for our Basket of Kittens schema definition
# File modules/mu/config/schema_helpers.rb, line 48 def self.schema @@schema end
Deep merge a configuration hash so we can meld different cloud providers' schemas together, while preserving documentation differences
# File modules/mu/config/schema_helpers.rb, line 54 def self.schemaMerge(orig, new, cloud) if new.is_a?(Hash) new.each_pair { |k, v| if cloud and k == "description" and v.is_a?(String) and !v.match(/\b#{Regexp.quote(cloud.upcase)}\b/) and !v.empty? new[k] = "+"+cloud.upcase+"+: "+v end if orig and orig.has_key?(k) elsif orig orig[k] = new[k] else orig = new end schemaMerge(orig[k], new[k], cloud) } elsif orig.is_a?(Array) and new orig.concat(new) orig.uniq! elsif new.is_a?(String) orig ||= "" orig += "\n" if !orig.empty? orig += "+#{cloud.upcase}+: "+new else # XXX I think this is a NOOP? end end
Make a deep copy of a config hash and pare it down to only primitive types, even at the leaves. @param config [Hash] @return [Hash]
# File modules/mu/config.rb, line 103 def self.stripConfig(config) MU::Config.manxify(Marshal.load(Marshal.dump(MU.structToHash(config.dup))), remove_runtime_keys: true) end
Accessor for tails in our Basket of Kittens. This should be a superset of user-supplied parameters. It also has machine-generated parameterized behaviors.
# File modules/mu/config.rb, line 67 def self.tails @@tails end
Public Instance Methods
Generate configuration for the general-purpose admin firewall rulesets (security groups in AWS). Note that these are unique to regions and individual VPCs (as well as Classic, which is just a degenerate case of a VPC
for our purposes. @param vpc [Hash]: A VPC
reference as defined in our config schema. This originates with the calling resource, so we'll peel out just what we need (a name or cloud id of a VPC
). @param admin_ip [String]: Optional string of an extra IP address to allow blanket access to the calling resource. @param cloud [String]: The parent resource's cloud plugin identifier @param region [String]: Cloud
provider region, if applicable. @return [Hash<String>]: A dependency description that the calling resource can then add to itself.
# File modules/mu/config/firewall_rule.rb, line 144 def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil, credentials: nil, rules_only: false) if !cloud or (cloud == "AWS" and !region) raise MuError, "Cannot call adminFirewallRuleset without specifying the parent's region and cloud provider" end hosts = Array.new hosts << "#{MU.my_public_ip}/32" if MU.my_public_ip hosts << "#{MU.my_private_ip}/32" if MU.my_private_ip hosts << "#{MU.mu_public_ip}/32" if MU.mu_public_ip hosts << "#{admin_ip}/32" if admin_ip hosts.uniq! rules = [] if cloud == "Google" rules = [ { "ingress" => true, "proto" => "all", "hosts" => hosts }, { "egress" => true, "proto" => "all", "hosts" => hosts } ] else rules = [ { "proto" => "tcp", "port_range" => "0-65535", "hosts" => hosts }, { "proto" => "udp", "port_range" => "0-65535", "hosts" => hosts }, { "proto" => "icmp", "port_range" => "-1", "hosts" => hosts } ] end if rules_only return rules end name = "admin" name += credentials.to_s if credentials realvpc = nil if vpc realvpc = {} ['vpc_name', 'vpc_id'].each { |p| if vpc[p] vpc[p.sub(/^vpc_/, '')] = vpc[p] vpc.delete(p) end } ['cloud', 'id', 'name', 'deploy_id', 'habitat', 'credentials'].each { |field| realvpc[field] = vpc[field] if !vpc[field].nil? } if !realvpc['id'].nil? and !realvpc['id'].empty? # Stupid kludge for Google cloud_ids which are sometimes URLs and # sometimes not. Requirements are inconsistent from scenario to # scenario. name = name + "-" + realvpc['id'].gsub(/.*\//, "") realvpc['id'] = getTail("id", value: realvpc['id'], prettyname: "Admin Firewall Ruleset #{name} Target VPC", cloudtype: "AWS::EC2::VPC::Id") if realvpc["id"].is_a?(String) elsif !realvpc['name'].nil? name = name + "-" + realvpc['name'] end end acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true, "credentials" => credentials } if cloud == "Google" and acl["vpc"] and acl["vpc"]["habitat"] acl['project'] = acl["vpc"]["habitat"]["id"] || acl["vpc"]["habitat"]["name"] end acl.delete("vpc") if !acl["vpc"] if !MU::Cloud.resourceClass(cloud, "FirewallRule").isGlobal? and !region.nil? and !region.empty? acl["region"] = region end @admin_firewall_rules << acl if !@admin_firewall_rules.include?(acl) return {"type" => "firewall_rule", "name" => name} end
For our resources which specify intra-stack dependencies, make sure those dependencies are actually declared.
# File modules/mu/config.rb, line 882 def check_dependencies ok = true @config.each_pair { |type, values| next if !values.instance_of?(Array) _shortclass, cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(type, false) next if !cfg_name values.each { |resource| next if !resource.kind_of?(Hash) or resource["dependencies"].nil? addme = [] deleteme = [] resource["dependencies"].each { |dependency| dependency["their_phase"] ||= dependency["phase"] dependency.delete("phase") dependency["my_phase"] ||= dependency["no_create_wait"] ? "groom" : "create" dependency.delete("no_create_wait") # make sure the thing we depend on really exists sibling = haveLitterMate?(dependency['name'], dependency['type']) if !sibling MU.log "Missing dependency: #{type}{#{resource['name']}} needs #{cfg_name}{#{dependency['name']}}", MU::ERR ok = false next end # Fudge dependency declarations to quash virtual_names that we know # are extraneous. Note that wee can't do all virtual names here; we # have no way to guess which of a collection of resources is the # real correct one. if sibling['virtual_name'] == dependency['name'] real_resources = [] found_exact = false resource["dependencies"].each { |dep_again| if dep_again['type'] == dependency['type'] and sibling['name'] == dep_again['name'] dependency['name'] = sibling['name'] found_exact = true break end } if !found_exact all_siblings = haveLitterMate?(dependency['name'], dependency['type'], has_multiple: true) if all_siblings.size > 0 all_siblings.each { |s| newguy = dependency.clone newguy['name'] = s['name'] addme << newguy } deleteme << dependency MU.log "Expanding dependency which maps to virtual resources to all matching real resources", MU::NOTICE, details: { sibling['virtual_name'] => addme } next end end end if dependency['their_phase'] == "groom" sibling['dependencies'].each { |sib_dep| next if sib_dep['type'] != cfg_name or sib_dep['their_phase'] != "groom" cousin = haveLitterMate?(sib_dep['name'], sib_dep['type']) if cousin and cousin['name'] == resource['name'] MU.log "Circular dependency between #{type} #{resource['name']} <=> #{dependency['type']} #{dependency['name']}", MU::ERR, details: [ resource['name'] => dependency, sibling['name'] => sib_dep ] ok = false end } end # Check for a circular relationship that will lead to a deadlock # when creating resource. This only goes one layer deep, and does # not consider groom-phase deadlocks. if dependency['their_phase'] == "groom" or dependency['my_phase'] == "groom" or ( !MU::Cloud.resourceClass(sibling['cloud'], type).deps_wait_on_my_creation and !MU::Cloud.resourceClass(resource['cloud'], type).waits_on_parent_completion ) next end if sibling['dependencies'] sibling['dependencies'].each { |sib_dep| next if sib_dep['type'] != cfg_name or sib_dep['my_phase'] == "groom" cousin = haveLitterMate?(sib_dep['name'], sib_dep['type']) if cousin and cousin['name'] == resource['name'] MU.log "Circular dependency between #{type} #{resource['name']} <=> #{dependency['type']} #{dependency['name']}", MU::ERR, details: [ resource['name'] => dependency, sibling['name'] => sib_dep ] ok = false end } end } resource["dependencies"].reject! { |dep| deleteme.include?(dep) } resource["dependencies"].concat(addme) resource["dependencies"].uniq! } } ok end
Instead of resolving a parameter, leave a placeholder for a cloud-specific variable that will be generated at runtime. Canonical use case: referring to a CloudFormation variable by reference, like “AWS::StackName” or “SomeChildTemplate.OutputVariableName.” @param code [String]: A string consistent of code which will be understood by the Cloud
layer, e.g. '“Ref” : “AWS::StackName”' (CloudFormation) @param placeholder [Object]: A placeholder value to use at the config parser stage, if the default string will not pass validation.
# File modules/mu/config.rb, line 156 def cloudCode(code, placeholder = "CLOUDCODEPLACEHOLDER") var_name = code.gsub(/[^a-z0-9]/i, "_") placeholder = code if placeholder.nil? getTail(var_name, value: placeholder, runtimecode: code) "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP" end
Take an IP block and split it into a more-or-less arbitrary number of subnets. @param ip_block [String]: CIDR of the network to subdivide @param subnets_desired [Integer]: Number of subnets we want back @param max_mask [Integer]: The highest netmask we're allowed to use for a subnet (various by cloud provider) @return [MU::Config::Tail]: Resulting subnet tails, or nil if an error occurred.
# File modules/mu/config/vpc.rb, line 1080 def divideNetwork(ip_block, subnets_desired, max_mask = 28) cidr = NetAddr::IPv4Net.parse(ip_block.to_s) # Ugly but reliable method of landing on the right subnet size subnet_bits = cidr.netmask.prefix_len begin subnet_bits += 1 if subnet_bits > max_mask MU.log "Can't subdivide #{cidr.to_s} into #{subnets_desired.to_s}", MU::ERR raise MuError, "Subnets smaller than /#{max_mask} not permitted" end end while cidr.subnet_count(subnet_bits) < subnets_desired if cidr.subnet_count(subnet_bits) > subnets_desired MU.log "Requested #{subnets_desired.to_s} subnets from #{cidr.to_s}, leaving #{(cidr.subnet_count(subnet_bits)-subnets_desired).to_s} unused /#{subnet_bits.to_s}s available", MU::NOTICE end begin subnets = [] (0..subnets_desired).each { |x| subnets << cidr.nth_subnet(subnet_bits, x).to_s } rescue RuntimeError => e if e.message.match(/exceeds subnets available for allocation/) MU.log e.message, MU::ERR MU.log "I'm attempting to create #{subnets_desired} subnets (one public and one private for each Availability Zone), of #{subnet_size} addresses each, but that's too many for a /#{cidr.netmask.prefix_len} network. Either declare a larger network, or explicitly declare a list of subnets with few enough entries to fit.", MU::ERR return nil else raise e end end subnets = getTail("subnetblocks", value: subnets.join(","), cloudtype: "CommaDelimitedList", description: "IP Address ranges to be used for VPC subnets", prettyname: "SubnetIpBlocks", list_of: "ip_block").map { |tail| tail["ip_block"] } subnets end
Wrapper method for creating a {MU::Config::Tail} object as a reference to a parameter that's valid in the loaded configuration. @param param [<String>]: The name of the parameter to which this should be tied. @param value [<String>]: The value of the parameter to return when asked @param prettyname [<String>]: A human-friendly parameter name to be used when generating CloudFormation templates and the like @param cloudtype [<String>]: A platform-specific identifier used by cloud layers to identify a parameter's type, e.g. AWS::EC2::VPC::Id @param valid_values [Array<String>]: A list of acceptable String values for the given parameter. @param description [<String>]: A long-form description of what the parameter does. @param list_of [<String>]: Indicates that the value should be treated as a member of a list (array) by the cloud layer. @param prefix [<String>]: A static String that should be prefixed to the stored value when queried @param suffix [<String>]: A static String that should be appended to the stored value when queried @param pseudo [<Boolean>]: This is a pseudo-parameter, automatically provided, and not available as user input. @param runtimecode [<String>]: Actual code to allow the cloud layer to interpret literally in its own idiom, e.g. '“Ref” : “AWS::StackName”' for CloudFormation
# File modules/mu/config/tail.rb, line 145 def getTail(param, value: nil, prettyname: nil, cloudtype: "String", valid_values: [], description: nil, list_of: nil, prefix: "", suffix: "", pseudo: false, runtimecode: nil) param = param.gsub(/[^a-z0-9_]/i, "_") if value.nil? if @@parameters.nil? or !@@parameters.has_key?(param) MU.log "Parameter '#{param}' (#{param.class.name}) referenced in config but not provided (#{caller[0]})", MU::DEBUG, details: @@parameters return nil # raise DeployParamError else value = @@parameters[param] end end if !prettyname.nil? prettyname.gsub!(/[^a-z0-9]/i, "") # comply with CloudFormation restrictions end if value.is_a?(MU::Config::Tail) MU.log "Parameter #{param} is using a nested parameter as a value. This rarely works, depending on the target cloud. YMMV.", MU::WARN tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode) elsif !list_of.nil? or (@@tails.has_key?(param) and @@tails[param].is_a?(Array)) tail = [] count = 0 value.split(/\s*,\s*/).each { |subval| if @@tails.has_key?(param) and !@@tails[param][count].nil? subval = @@tails[param][count].values.first.to_s if subval.nil? list_of = @@tails[param][count].values.first.getName if list_of.nil? prettyname = @@tails[param][count].values.first.getPrettyName if prettyname.nil? description = @@tails[param][count].values.first.description if description.nil? valid_values = @@tails[param][count].values.first.valid_values if valid_values.nil? or valid_values.empty? cloudtype = @@tails[param][count].values.first.getCloudType if @@tails[param][count].values.first.getCloudType != "String" end prettyname = param.capitalize if prettyname.nil? tail << { list_of => MU::Config::Tail.new(list_of, subval, prettyname, cloudtype, valid_values, description, true, pseudo: pseudo, index: count) } count = count + 1 } else if @@tails.has_key?(param) pseudo = @@tails[param].pseudo value = @@tails[param].to_s if value.nil? prettyname = @@tails[param].getPrettyName if prettyname.nil? description = @@tails[param].description if description.nil? valid_values = @@tails[param].valid_values if valid_values.nil? or valid_values.empty? cloudtype = @@tails[param].getCloudType if @@tails[param].getCloudType != "String" end tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode) end if valid_values and valid_values.size > 0 and value if !valid_values.include?(value) raise DeployParamError, "Invalid parameter value '#{value}' supplied for '#{param}'" end end @@tails[param] = tail tail end
See if a given resource is configured in the current stack @param name [String]: The name of the resource being checked @param type [String]: The type of resource being checked @return [Boolean]
# File modules/mu/config.rb, line 470 def haveLitterMate?(name, type, has_multiple: false) @kittencfg_semaphore.synchronize { matches = [] _shortclass, _cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type) if @kittens[cfg_plural] @kittens[cfg_plural].each { |kitten| if kitten['name'].to_s == name.to_s or kitten['virtual_name'].to_s == name.to_s or (has_multiple and name.nil?) if has_multiple matches << kitten else return kitten end end } end if has_multiple return matches else return false end } end
Insert a resource into the current stack @param descriptor [Hash]: The configuration description, as from a Basket of Kittens @param type [String]: The type of resource being added @param delay_validation [Boolean]: Whether to hold off on calling the resource's validateConfig method @param ignore_duplicates [Boolean]: Do not raise an exception if we attempt to insert a resource with a name
field that's already in use
# File modules/mu/config.rb, line 519 def insertKitten(descriptor, type, delay_validation = false, ignore_duplicates: false, overwrite: false) append = false start = Time.now shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type) MU.log "insertKitten on #{cfg_name} #{descriptor['name']} (delay_validation: #{delay_validation.to_s})", MU::DEBUG, details: caller[0] if overwrite removeKitten(descriptor['name'], type) end if !ignore_duplicates and haveLitterMate?(descriptor['name'], cfg_name) # raise DuplicateNameError, "A #{shortclass} named #{descriptor['name']} has already been inserted into this configuration" end @kittencfg_semaphore.synchronize { append = !@kittens[cfg_plural].include?(descriptor) # Skip if this kitten has already been validated and appended if !append and descriptor["#MU_VALIDATED"] return true end } ok = true if descriptor['cloud'] and !MU::Cloud.availableClouds.include?(descriptor['cloud']) if MU::Cloud.supportedClouds.include?(descriptor['cloud']) MU.log "#{cfg_name} #{descriptor['name']} is configured with cloud #{descriptor['cloud']}, but no #{descriptor['cloud']} credentials available", MU::ERR else MU.log "#{cfg_name} #{descriptor['name']}: Cloud provider #{descriptor['cloud']} is not supported", MU::ERR, details: MU::Cloud.supportedClouds end return false end descriptor["#MU_CLOUDCLASS"] = classname applyInheritedDefaults(descriptor, cfg_plural) # Meld defaults from our global schema and, if applicable, from our # cloud-specific schema. schemaclass = Object.const_get("MU").const_get("Config").const_get(shortclass) myschema = Marshal.load(Marshal.dump(MU::Config.schema["properties"][cfg_plural]["items"])) more_required, more_schema = MU::Cloud.resourceClass(descriptor["cloud"], type).schema(self) if more_schema MU::Config.schemaMerge(myschema["properties"], more_schema, descriptor["cloud"]) end myschema["required"] ||= [] if more_required myschema["required"].concat(more_required) myschema["required"].uniq! end descriptor = applySchemaDefaults(descriptor, myschema, type: shortclass) MU.log "Schema check on #{descriptor['cloud']} #{cfg_name} #{descriptor['name']}", MU::DEBUG, details: myschema if (descriptor["region"] and descriptor["region"].empty?) or (descriptor['cloud'] == "Google" and ["firewall_rule", "vpc"].include?(cfg_name)) descriptor.delete("region") end # Make sure a sensible region has been targeted, if applicable classobj = MU::Cloud.cloudClass(descriptor["cloud"]) if descriptor["region"] valid_regions = classobj.listRegions if !valid_regions.include?(descriptor["region"]) MU.log "Known regions for cloud '#{descriptor['cloud']}' do not include '#{descriptor["region"]}'", MU::ERR, details: valid_regions ok = false end end if descriptor.has_key?('project') if descriptor['project'].nil? descriptor.delete('project') elsif haveLitterMate?(descriptor['project'], "habitats") MU::Config.addDependency(descriptor, descriptor['project'], "habitat") end end # Does this resource go in a VPC? if !descriptor["vpc"].nil? and !delay_validation # Quietly fix old vpc reference style if descriptor['vpc']['vpc_id'] descriptor['vpc']['id'] ||= descriptor['vpc']['vpc_id'] descriptor['vpc'].delete('vpc_id') end if descriptor['vpc']['vpc_name'] descriptor['vpc']['name'] = descriptor['vpc']['vpc_name'] descriptor['vpc'].delete('vpc_name') end descriptor['vpc']['cloud'] = descriptor['cloud'] if descriptor['credentials'] descriptor['vpc']['credentials'] ||= descriptor['credentials'] end if descriptor['vpc']['region'].nil? and !descriptor['region'].nil? and !descriptor['region'].empty? and descriptor['vpc']['cloud'] != "Google" descriptor['vpc']['region'] = descriptor['region'] end # If we're using a VPC in this deploy, set it as a dependency if !descriptor["vpc"]["name"].nil? and haveLitterMate?(descriptor["vpc"]["name"], "vpcs") and descriptor["vpc"]['deploy_id'].nil? and descriptor["vpc"]['id'].nil? and !(cfg_name == "vpc" and descriptor['name'] == descriptor['vpc']['name']) MU::Config.addDependency(descriptor, descriptor['vpc']['name'], "vpc") siblingvpc = haveLitterMate?(descriptor["vpc"]["name"], "vpcs") if siblingvpc and siblingvpc['bastion'] and ["server", "server_pool", "container_cluster"].include?(cfg_name) and !descriptor['bastion'] if descriptor['name'] != siblingvpc['bastion']['name'] MU::Config.addDependency(descriptor, siblingvpc['bastion']['name'], "server") end end # things that live in subnets need their VPCs to be fully # resolved before we can proceed if ["server", "server_pool", "loadbalancer", "database", "cache_cluster", "container_cluster", "storage_pool"].include?(cfg_name) if !siblingvpc["#MU_VALIDATED"] ok = false if !insertKitten(siblingvpc, "vpcs", overwrite: overwrite) end end if !MU::Config::VPC.processReference(descriptor['vpc'], cfg_plural, descriptor, self, dflt_region: descriptor['region'], credentials: descriptor['credentials'], dflt_project: descriptor['project'], sibling_vpcs: @kittens['vpcs']) ok = false end # If we're using a VPC from somewhere else, make sure the flippin' # thing exists, and also fetch its id now so later search routines # don't have to work so hard. else if !MU::Config::VPC.processReference(descriptor["vpc"], cfg_plural, descriptor, self, credentials: descriptor['credentials'], dflt_project: descriptor['project'], dflt_region: descriptor['region']) ok = false end end # if we didn't specify credentials but can inherit some from our target # VPC, do so if descriptor["vpc"]["credentials"] descriptor["credentials"] ||= descriptor["vpc"]["credentials"] end # Clean crud out of auto-created VPC declarations so they don't trip # the schema validator when it's invoked later. if !["server", "server_pool", "database"].include?(cfg_name) descriptor['vpc'].delete("nat_ssh_user") end if descriptor['vpc']['cloud'] == "Google" descriptor['vpc'].delete("region") end if ["firewall_rule", "function"].include?(cfg_name) descriptor['vpc'].delete("subnet_pref") end end # Does it have generic ingress rules? fwname = cfg_name+descriptor['name'] if (descriptor['ingress_rules'] or ["server", "server_pool", "database", "cache_cluster"].include?(cfg_name)) descriptor['ingress_rules'] ||= [] acl = haveLitterMate?(fwname, "firewall_rules") already_exists = !acl.nil? acl ||= { "name" => fwname, "rules" => descriptor['ingress_rules'], "region" => descriptor['region'], "credentials" => descriptor["credentials"] } if !MU::Cloud.resourceClass(descriptor["cloud"], "FirewallRule").isGlobal? acl['region'] = descriptor['region'] acl['region'] ||= classobj.myRegion(acl['credentials']) else acl.delete("region") end if descriptor["vpc"] acl["vpc"] = descriptor['vpc'].dup acl["vpc"].delete("subnet_pref") end ["optional_tags", "tags", "cloud", "project"].each { |param| acl[param] = descriptor[param] if descriptor[param] } descriptor["add_firewall_rules"] ||= [] descriptor["add_firewall_rules"] << {"name" => fwname, "type" => "firewall_rules" } # XXX why the duck is there a type argument required here? descriptor["add_firewall_rules"].uniq! acl = resolveIntraStackFirewallRefs(acl, delay_validation) ok = false if !insertKitten(acl, "firewall_rules", delay_validation, overwrite: already_exists) end # Does it declare association with any sibling LoadBalancers? if !descriptor["loadbalancers"].nil? descriptor["loadbalancers"].each { |lb| if !lb["concurrent_load_balancer"].nil? MU::Config.addDependency(descriptor, lb["concurrent_load_balancer"], "loadbalancer") end } end # Does it want to know about Storage Pools? if !descriptor["storage_pools"].nil? descriptor["storage_pools"].each { |sp| if sp["name"] MU::Config.addDependency(descriptor, sp["name"], "storage_pool") end } end # Does it declare association with first-class firewall_rules? if !descriptor["add_firewall_rules"].nil? descriptor["add_firewall_rules"].each { |acl_include| next if !acl_include["name"] and !acl_include["rule_name"] acl_include["name"] ||= acl_include["rule_name"] if haveLitterMate?(acl_include["name"], "firewall_rules") MU::Config.addDependency(descriptor, acl_include["name"], "firewall_rule", my_phase: ((cfg_name == "vpc") ? "groom" : "create")) elsif acl_include["name"] MU.log shortclass.to_s+" #{descriptor['name']} depends on FirewallRule #{acl_include["name"]}, but no such rule declared.", MU::ERR ok = false end } end # Does it declare some alarms? if descriptor["alarms"] && !descriptor["alarms"].empty? descriptor["alarms"].each { |alarm| alarm["name"] = "#{cfg_name}-#{descriptor["name"]}-#{alarm["name"]}" alarm['dimensions'] ||= [] alarm["namespace"] ||= descriptor['name'] alarm["credentials"] = descriptor["credentials"] alarm["#TARGETCLASS"] = cfg_name alarm["#TARGETNAME"] = descriptor['name'] alarm['cloud'] = descriptor['cloud'] ok = false if !insertKitten(alarm, "alarms", true, overwrite: overwrite) } descriptor.delete("alarms") end # Does it want to meld another deployment's resources into its metadata? if !descriptor["existing_deploys"].nil? and !descriptor["existing_deploys"].empty? descriptor["existing_deploys"].each { |ext_deploy| if ext_deploy["cloud_type"].nil? MU.log "You must provide a cloud_type", MU::ERR ok = false end if ext_deploy["cloud_id"] found = MU::MommaCat.findStray( descriptor['cloud'], ext_deploy["cloud_type"], cloud_id: ext_deploy["cloud_id"], region: descriptor['region'], dummy_ok: false ).first if found.nil? MU.log "Couldn't find existing #{ext_deploy["cloud_type"]} resource #{ext_deploy["cloud_id"]}", MU::ERR ok = false end elsif ext_deploy["mu_name"] && ext_deploy["deploy_id"] found = MU::MommaCat.findStray( descriptor['cloud'], ext_deploy["cloud_type"], deploy_id: ext_deploy["deploy_id"], mu_name: ext_deploy["mu_name"], region: descriptor['region'], dummy_ok: false ).first if found.nil? MU.log "Couldn't find existing #{ext_deploy["cloud_type"]} resource - #{ext_deploy["mu_name"]} / #{ext_deploy["deploy_id"]}", MU::ERR ok = false end else MU.log "Trying to find existing deploy, but either the cloud_id is not valid or no mu_name and deploy_id where provided", MU::ERR ok = false end } end if !delay_validation # Call the generic validation for the resource type, first and foremost # XXX this might have to be at the top of this insertKitten instead of # here ok = false if !schemaclass.validate(descriptor, self) plain_cfg = MU::Config.stripConfig(descriptor) plain_cfg.delete("#MU_CLOUDCLASS") plain_cfg.delete("#MU_VALIDATION_ATTEMPTED") plain_cfg.delete("#TARGETCLASS") plain_cfg.delete("#TARGETNAME") plain_cfg.delete("parent_block") if cfg_plural == "vpcs" begin JSON::Validator.validate!(myschema, plain_cfg) rescue JSON::Schema::ValidationError pp plain_cfg # Use fully_validate to get the complete error list, save some time errors = JSON::Validator.fully_validate(myschema, plain_cfg) realerrors = [] errors.each { |err| if !err.match(/The property '.+?' of type MU::Config::Tail did not match the following type:/) realerrors << err end } if realerrors.size > 0 MU.log "Validation error on #{descriptor['cloud']} #{cfg_name} #{descriptor['name']} (insertKitten called from #{caller[1]} with delay_validation=#{delay_validation}) #{@@config_path}!\n"+realerrors.join("\n"), MU::ERR, details: descriptor raise ValidationError, "Validation error on #{descriptor['cloud']} #{cfg_name} #{descriptor['name']} #{@@config_path}!\n"+realerrors.join("\n") end end # Run the cloud class's deeper validation, unless we've already failed # on stuff that will cause spurious alarms further in if ok parser = MU::Cloud.resourceClass(descriptor['cloud'], type) original_descriptor = MU::Config.stripConfig(descriptor) passed = parser.validateConfig(descriptor, self) if !passed descriptor = original_descriptor ok = false end # Make sure we've been configured with the right credentials cloudbase = MU::Cloud.cloudClass(descriptor['cloud']) credcfg = cloudbase.credConfig(descriptor['credentials']) if !credcfg or credcfg.empty? raise ValidationError, "#{descriptor['cloud']} #{cfg_name} #{descriptor['name']} declares credential set #{descriptor['credentials']}, but no such credentials exist for that cloud provider" end descriptor['#MU_VALIDATED'] = true end end descriptor["dependencies"].uniq! if descriptor["dependencies"] @kittencfg_semaphore.synchronize { @kittens[cfg_plural] << descriptor if append } MU.log "insertKitten completed #{cfg_name} #{descriptor['name']} in #{sprintf("%.2fs", Time.now-start)}", MU::DEBUG ok end
Catch calls to missing variables in Basket of Kittens files when being parsed by ERB, and replace with placeholders for parameters. This method_missing
is only defined innside {MU::Config.resolveConfig}
# File modules/mu/config.rb, line 120 def method_missing(var_name) if @param_pass "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP" else tail = getTail(var_name.to_s) if tail.is_a?(Array) if @param_pass return tail.map {|f| f.values.first.to_s }.join(",") else # Don't try to jam complex types into a string file format, just # sub them back in later from a placeholder. return "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP" end else if @param_pass tail.to_s else return "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP" end end end end
A check for the existence of a user-supplied parameter value that can be easily run in an ERB block in a Basket of Kittens.
# File modules/mu/config.rb, line 146 def parameter?(var_name) @@user_supplied_parameters.has_key?(var_name) end
Remove a resource from the current stack @param name [String]: The name of the resource being removed @param type [String]: The type of resource being removed
# File modules/mu/config.rb, line 498 def removeKitten(name, type) @kittencfg_semaphore.synchronize { _shortclass, _cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type) deletia = nil if @kittens[cfg_plural] @kittens[cfg_plural].each { |kitten| if kitten['name'] == name deletia = kitten break end } @kittens[type].delete(deletia) if !deletia.nil? end } end
Load up our YAML or JSON and parse it through ERB, optionally substituting externally-supplied parameters.
# File modules/mu/config.rb, line 109 def resolveConfig(path: @@config_path, param_pass: false, cloud: nil) config = nil @param_pass = param_pass if cloud MU.log "Exposing cloud variable to ERB with value of #{cloud}", MU::DEBUG end # Catch calls to missing variables in Basket of Kittens files when being # parsed by ERB, and replace with placeholders for parameters. This # method_missing is only defined innside {MU::Config.resolveConfig} def method_missing(var_name) if @param_pass "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP" else tail = getTail(var_name.to_s) if tail.is_a?(Array) if @param_pass return tail.map {|f| f.values.first.to_s }.join(",") else # Don't try to jam complex types into a string file format, just # sub them back in later from a placeholder. return "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP" end else if @param_pass tail.to_s else return "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP" end end end end # A check for the existence of a user-supplied parameter value that can # be easily run in an ERB block in a Basket of Kittens. def parameter?(var_name) @@user_supplied_parameters.has_key?(var_name) end # Instead of resolving a parameter, leave a placeholder for a # cloud-specific variable that will be generated at runtime. Canonical # use case: referring to a CloudFormation variable by reference, like # "AWS::StackName" or "SomeChildTemplate.OutputVariableName." # @param code [String]: A string consistent of code which will be understood by the Cloud layer, e.g. '"Ref" : "AWS::StackName"' (CloudFormation) # @param placeholder [Object]: A placeholder value to use at the config parser stage, if the default string will not pass validation. def cloudCode(code, placeholder = "CLOUDCODEPLACEHOLDER") var_name = code.gsub(/[^a-z0-9]/i, "_") placeholder = code if placeholder.nil? getTail(var_name, value: placeholder, runtimecode: code) "MU::Config.getTail PLACEHOLDER #{var_name} REDLOHECALP" end # Make sure our parameter values are all available in the local namespace # that ERB will be using, minus any that conflict with existing variables erb_binding = get_binding(@@tails.keys.sort) @@tails.each_pair { |key, tail| next if !tail.is_a?(MU::Config::Tail) or tail.is_list_element # XXX figure out what to do with lists begin erb_binding.local_variable_set(key.to_sym, tail.to_s) rescue NameError MU.log "Binding #{key} = #{tail.to_s}", MU::DEBUG erb_binding.local_variable_set(key.to_sym, tail.to_s) end } # Figure out what kind of file we're loading. We handle includes # differently if YAML is involved. These globals get used inside # templates. They're globals on purpose. Stop whining. $file_format = MU::Config.guessFormat(path) $yaml_refs = {} erb = ERB.new(File.read(path), nil, "<>") erb.filename = path begin raw_text = erb.result(erb_binding) rescue NameError => e loc = e.backtrace[0].sub(/:(\d+):.*/, ':\1') msg = if e.message.match(/wrong constant name Config.getTail PLACEHOLDER ([^\s]+) REDLOHECALP/) "Variable '#{Regexp.last_match[1]}' referenced in config, but not defined. Missing required parameter?" else e.message end raise ValidationError, msg+" at "+loc end raw_json = nil # If we're working in YAML, do some magic to make includes work better. yaml_parse_error = nil if $file_format == :yaml begin raw_json = JSON.generate(YAML.load(MU::Config.resolveYAMLAnchors(raw_text))) rescue Psych::SyntaxError => e raw_json = raw_text yaml_parse_error = e.message end else raw_json = raw_text end begin config = JSON.parse(raw_json) if @@parameters['cloud'] config['cloud'] ||= @@parameters['cloud'].to_s end if param_pass and config.is_a?(Hash) config.keys.each { |key| if key != "parameters" if key == "appname" and @@parameters["myAppName"].nil? $myAppName = config["appname"].upcase.dup $myAppName.freeze @@parameters["myAppName"] = getTail("myAppName", value: config["appname"].upcase, pseudo: true).to_s end config.delete(key) end } elsif config.is_a?(Hash) config.delete("parameters") end rescue JSON::ParserError => e badconf = File.new("/tmp/badconf.#{$$}", File::CREAT|File::TRUNC|File::RDWR, 0400) badconf.puts raw_text badconf.close if !yaml_parse_error.nil? and !path.match(/\.json/) MU.log "YAML Error parsing #{path}! Complete file dumped to /tmp/badconf.#{$$}", MU::ERR, details: yaml_parse_error else MU.log "JSON Error parsing #{path}! Complete file dumped to /tmp/badconf.#{$$}", MU::ERR, details: e.message end raise ValidationError end undef :method_missing return [MU::Config.fixDashes(config), raw_text] end
FirewallRules can reference other FirewallRules, which means we need to do an extra pass to make sure we get all intra-stack dependencies correct. @param acl [Hash]: The configuration hash for the FirewallRule
to check @return [Hash]
# File modules/mu/config/firewall_rule.rb, line 117 def resolveIntraStackFirewallRefs(acl, delay_validation = false) acl["rules"].each { |acl_include| if acl_include['sgs'] acl_include['sgs'].each { |sg_ref| if haveLitterMate?(sg_ref, "firewall_rules") MU::Config.addDependency(acl, sg_ref, "firewall_rule", my_phase: "groom") siblingfw = haveLitterMate?(sg_ref, "firewall_rules") if !siblingfw["#MU_VALIDATED"] # XXX raise failure somehow insertKitten(siblingfw, "firewall_rules", delay_validation: delay_validation) end end } end } acl end
Convert parameter entries that constitute whole config keys into {MU::Config::Tail} objects.
# File modules/mu/config.rb, line 374 def resolveTails(tree, indent= "") if tree.is_a?(Hash) tree.each_pair { |key, val| tree[key] = resolveTails(val, indent+" ") } elsif tree.is_a?(Array) newtree = [] tree.each { |item| newtree << resolveTails(item, indent+" ") } tree = newtree elsif tree.is_a?(String) and tree.match(/^(.*?)MU::Config.getTail PLACEHOLDER (.+?) REDLOHECALP(.*)/) tree = getTail($2, prefix: $1, suffix: $3) if tree.nil? and @@tails.has_key?($2) # XXX why necessary? tree = @@tails[$2] end end return tree end
Output the dependencies of this BoK stack as a directed acyclic graph. Very useful for debugging.
# File modules/mu/config/doc_helpers.rb, line 102 def visualizeDependencies # GraphViz won't like MU::Config::Tail, pare down to plain Strings config = MU::Config.stripConfig(@config) begin g = GraphViz.new(:G, :type => :digraph) # Generate a GraphViz node for each resource in this stack nodes = {} MU::Cloud.resource_types.each_pair { |classname, attrs| nodes[attrs[:cfg_name]] = {} if config.has_key?(attrs[:cfg_plural]) and config[attrs[:cfg_plural]] config[attrs[:cfg_plural]].each { |resource| nodes[attrs[:cfg_name]][resource['name']] = g.add_nodes("#{classname}: #{resource['name']}") } end } # Now add edges corresponding to the dependencies they list MU::Cloud.resource_types.values.each { |attrs| if config.has_key?(attrs[:cfg_plural]) and config[attrs[:cfg_plural]] config[attrs[:cfg_plural]].each { |resource| if resource.has_key?("dependencies") me = nodes[attrs[:cfg_name]][resource['name']] resource["dependencies"].each { |dep| parent = nodes[dep['type']][dep['name']] g.add_edges(me, parent) } end } end } # Spew some output? MU.log "Emitting dependency graph as /tmp/#{config['appname']}.jpg", MU::NOTICE g.output(:jpg => "/tmp/#{config['appname']}.jpg") rescue StandardError => e MU.log "Failed to generate GraphViz dependency tree: #{e.inspect}. This should only matter to developers.", MU::WARN, details: e.backtrace end end
Private Instance Methods
Given a bare hash describing a resource, insert default values which can be inherited from its parent or from the root of the BoK. @param kitten [Hash]: A resource descriptor @param type [String]: The type of resource this is (“servers” etc)
# File modules/mu/config/schema_helpers.rb, line 338 def applyInheritedDefaults(kitten, type) return if !kitten.is_a?(Hash) kitten['cloud'] ||= @config['cloud'] kitten['cloud'] ||= MU::Config.defaultCloud if !MU::Cloud.supportedClouds.include?(kitten['cloud']) return end cloudclass = MU::Cloud.cloudClass(kitten['cloud']) resclass = MU::Cloud.resourceClass(kitten['cloud'], type) schema_fields = ["us_only", "scrub_mu_isms", "credentials", "billing_acct"] if !resclass.isGlobal? kitten['region'] ||= @config['region'] kitten['region'] ||= cloudclass.myRegion(kitten['credentials']) schema_fields << "region" end kitten['credentials'] ||= @config['credentials'] kitten['credentials'] ||= cloudclass.credConfig(name_only: true) kitten['us_only'] ||= @config['us_only'] kitten['us_only'] ||= false kitten['scrub_mu_isms'] ||= @config['scrub_mu_isms'] kitten['scrub_mu_isms'] ||= false if kitten['cloud'] == "Google" # TODO this should be cloud-generic (handle AWS accounts, Azure subscriptions) if resclass.canLiveIn.include?(:Habitat) kitten["project"] ||= MU::Cloud::Google.defaultProject(kitten['credentials']) schema_fields << "project" end if kitten['region'].nil? and !kitten['#MU_CLOUDCLASS'].nil? and !resclass.isGlobal? and ![MU::Cloud::VPC, MU::Cloud::FirewallRule].include?(kitten['#MU_CLOUDCLASS']) if MU::Cloud::Google.myRegion((kitten['credentials'])).nil? raise ValidationError, "Google '#{type}' resource '#{kitten['name']}' declared without a region, but no default Google region declared in mu.yaml under #{kitten['credentials'].nil? ? "default" : kitten['credentials']} credential set" end kitten['region'] ||= MU::Cloud::Google.myRegion end elsif kitten["cloud"] == "AWS" and !resclass.isGlobal? and !kitten['region'] if MU::Cloud::AWS.myRegion.nil? raise ValidationError, "AWS resource declared without a region, but no default AWS region found" end kitten['region'] ||= MU::Cloud::AWS.myRegion end kitten['billing_acct'] ||= @config['billing_acct'] if @config['billing_acct'] kitten["dependencies"] ||= [] # Make sure the schema knows about these "new" fields, so that validation # doesn't trip over them. schema_fields.each { |field| if @@schema["properties"][field] MU.log "Adding #{field} to schema for #{type} #{kitten['cloud']}", MU::DEBUG, details: @@schema["properties"][field] @@schema["properties"][type]["items"]["properties"][field] ||= @@schema["properties"][field] end } end
# File modules/mu/config/schema_helpers.rb, line 276 def applySchemaDefaults(conf_chunk = config, schema_chunk = schema, depth = 0, siblings = nil, type: nil) return if schema_chunk.nil? if conf_chunk != nil and schema_chunk["properties"].kind_of?(Hash) and conf_chunk.is_a?(Hash) if schema_chunk["properties"]["creation_style"].nil? or schema_chunk["properties"]["creation_style"] != "existing" schema_chunk["properties"].each_pair { |key, subschema| shortclass = if conf_chunk[key] shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(key, false) shortclass else nil end new_val = applySchemaDefaults(conf_chunk[key], subschema, depth+1, conf_chunk, type: shortclass).dup if !new_val.nil? begin conf_chunk[key] = Marshal.load(Marshal.dump(new_val)) rescue TypeError conf_chunk[key] = new_val.clone end end } end elsif schema_chunk["type"] == "array" and conf_chunk.kind_of?(Array) conf_chunk.map! { |item| # If we're working on a resource type, go get implementation-specific # schema information so that we set those defaults correctly. realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"] and MU::Cloud.supportedClouds.include?(item['cloud']) _toplevel_required, cloudschema = MU::Cloud.resourceClass(item["cloud"], type).schema(self) newschema = schema_chunk["items"].dup MU::Config.schemaMerge(newschema["properties"], cloudschema, item["cloud"]) newschema else schema_chunk["items"].dup end applySchemaDefaults(item, realschema, depth+1, conf_chunk, type: type).dup } else if conf_chunk.nil? and !schema_chunk["default_if"].nil? and !siblings.nil? schema_chunk["default_if"].each { |cond| if siblings[cond["key_is"]] == cond["value_is"] return Marshal.load(Marshal.dump(cond["set"])) end } end if conf_chunk.nil? and schema_chunk["default"] != nil return Marshal.load(Marshal.dump(schema_chunk["default"])) end end return conf_chunk end
Namespace magic to pass to ERB's result method.
# File modules/mu/config.rb, line 1159 def get_binding(keyset) environment = $environment myPublicIp = $myPublicIp myRoot = $myRoot myAZ = $myAZ myRegion = $myRegion myAppName = $myAppName # return MU::Config.global_bindings[keyset] if MU::Config.global_bindings[keyset] MU::Config.global_bindings[keyset] = binding MU::Config.global_bindings[keyset] end
(see include
)
# File modules/mu/config.rb, line 1154 def include(file) MU::Config.include(file, get_binding(@@tails.keys.sort), @param_pass) end
# File modules/mu/config.rb, line 1172 def validate(config = @config) ok = true count = 0 @kittens ||= {} types = MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] } types.each { |type| @kittens[type] = config[type] @kittens[type] ||= [] @kittens[type].each { |k| applyInheritedDefaults(k, type) } count = count + @kittens[type].size } if count == 0 MU.log "You must declare at least one resource to create", MU::ERR ok = false end @nat_routes ||= {} types.each { |type| @kittens[type].each { |descriptor| ok = false if !insertKitten(descriptor, type) } } newrules = [] @kittens["firewall_rules"].each { |acl| newrules << resolveIntraStackFirewallRefs(acl) } @kittens["firewall_rules"] = newrules # VPCs do complex things in their cloud-layer validation that other # resources tend to need, like subnet allocation, so hit them early. @kittens["vpcs"].each { |vpc| ok = false if !insertKitten(vpc, "vpcs") } # Make sure validation has been called for all on-the-fly generated # resources. validated_something_new = false begin validated_something_new = false types.each { |type| @kittens[type].each { |descriptor| if !descriptor["#MU_VALIDATION_ATTEMPTED"] validated_something_new = true ok = false if !insertKitten(descriptor, type) descriptor["#MU_VALIDATION_ATTEMPTED"] = true end } } end while validated_something_new # Do another pass of resolving intra-stack VPC peering, in case an # early-parsing VPC needs more details from a later-parsing one @kittens["vpcs"].each { |vpc| ok = false if !MU::Config::VPC.resolvePeers(vpc, self) } # add some default holes to allow dependent instances into databases @kittens["databases"].each { |db| if db['port'].nil? db['port'] = 3306 if ["mysql", "aurora"].include?(db['engine']) db['port'] = 5432 if ["postgres"].include?(db['engine']) db['port'] = 1433 if db['engine'].match(/^sqlserver\-/) db['port'] = 1521 if db['engine'].match(/^oracle\-/) end ruleset = haveLitterMate?("database"+db['name'], "firewall_rules") if ruleset ["server_pools", "servers"].each { |type| _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type) @kittens[cfg_plural].each { |server| server["dependencies"].each { |dep| if dep["type"] == "database" and dep["name"] == db["name"] # XXX this is AWS-specific, I think. We need to use source_tags to make this happen in Google. This logic probably needs to be dumped into the database layer. ruleset["rules"] << { "proto" => "tcp", "port" => db["port"], "sgs" => [cfg_name+server['name']] } MU::Config.addDependency(ruleset, cfg_name+server['name'], "firewall_rule", my_phase: "groom") end } } } end } seen = [] # XXX seem to be not detecting duplicate admin firewall_rules in adminFirewallRuleset @admin_firewall_rules.each { |acl| next if seen.include?(acl['name']) ok = false if !insertKitten(acl, "firewall_rules") seen << acl['name'] } types.each { |type| config[type] = @kittens[type] if @kittens[type].size > 0 } ok = false if !check_dependencies # TODO enforce uniqueness of resource names raise ValidationError if !ok # XXX Does commenting this out make sense? Do we want to apply it to top-level # keys and ignore resources, which validate when insertKitten is called now? # begin # JSON::Validator.validate!(MU::Config.schema, plain_cfg) # rescue JSON::Schema::ValidationError => e # # Use fully_validate to get the complete error list, save some time # errors = JSON::Validator.fully_validate(MU::Config.schema, plain_cfg) # realerrors = [] # errors.each { |err| # if !err.match(/The property '.+?' of type MU::Config::Tail did not match the following type:/) # realerrors << err # end # } # if realerrors.size > 0 # raise ValidationError, "Validation error in #{@@config_path}!\n"+realerrors.join("\n") # end # end end