class MU::Adoption

Scrape cloud providers for existing resources, and reverse-engineer them into runnable {MU::Config} descriptors and/or {MU::MommaCat} deploy objects.

Constants

GROUPMODES

Presets methods we use to clump discovered resources into discrete deploys

Attributes

found[R]

Public Class Methods

new(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: false, diff: false, habitats: [], scrub_mu_isms: false, regions: [], merge: false, pattern: nil) click to toggle source
# File modules/mu/adoption.rb, line 33
def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: false, diff: false, habitats: [], scrub_mu_isms: false, regions: [], merge: false, pattern: nil)
  @scraped = {}
  @clouds = clouds
  @types = types
  @parent = parent
  @boks = {}
  @billing = billing
  @reference_map = {}
  @sources = sources
  @target_creds = credentials
  @group_by = group_by
  @savedeploys = savedeploys
  @diff = diff
  @habitats = habitats
  @regions = regions
  @habitats ||= []
  @scrub_mu_isms = scrub_mu_isms
  @merge = merge
  @pattern = pattern
end
tagsToName(tags = [], basename: nil) click to toggle source

Given a list of BoK style tags, try to reverse-engineer the correct Basket of Kittens shorthand name of the resource that owns them. Mostly this infers from Mu-style tagging, but we'll add a couple cases for special cloud provider cases. @param tags [Array<Hash>] @param basename [String] return [String]

# File modules/mu/adoption.rb, line 159
def self.tagsToName(tags = [], basename: nil)
  tags.each { |tag|
    if tag['key'] == "aws:cloudformation:logical-id"
      return tag['value']
    end
  }
  muid = nil
  tags.each { |tag|
    if tag['key'] == "MU-ID" or tag['key'] == "mu-id"
      muid = tag['value']
      break
    end
  }

  tags.each { |tag|
    if tag['key'] == "Name"
      if muid and tag['value'].match(/^#{Regexp.quote(muid)}-(.*)/)
        return Regexp.last_match[1].downcase
      else
        return tag['value'].downcase
      end
    end
  }

  if basename and muid and basename.match(/^#{Regexp.quote(muid)}-(.*)/)
    return Regexp.last_match[1].downcase
  end

  nil
end

Private Class Methods

deDuplicateName(kitten_cfg, res_class) click to toggle source
# File modules/mu/adoption.rb, line 938
def self.deDuplicateName(kitten_cfg, res_class)
  orig_name = kitten_cfg['name'].dup
  if kitten_cfg['parent'] and kitten_cfg['parent'].respond_to?(:id) and kitten_cfg['parent'].id
    kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['parent'].id
  elsif kitten_cfg['project']
    kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['project']
  elsif kitten_cfg['region']
    kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['region']
  elsif kitten_cfg['cloud_id']
    kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['cloud_id'].gsub(/[^a-z0-9]/i, "-")
  else
    raise MU::Config::DuplicateNameError, "Saw duplicate #{res_class.cfg_name} name #{orig_name} and couldn't come up with a good way to differentiate them"
  end
end

Public Instance Methods

generateBaskets(prefix: "") click to toggle source

Generate a {MU::Config} (Basket of Kittens) hash using our discovered cloud objects. @return [Hash]

# File modules/mu/adoption.rb, line 193
    def generateBaskets(prefix: "")
      groupings = {
        "" =>  MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }
      }

      # XXX as soon as we come up with a method that isn't about what resource
      # type you are, this code will stop making sense
      if @group_by == :logical
        groupings = {
          "spaces" => ["folders", "habitats"],
          "people" => ["users", "groups", "roles"],
          "network" => ["vpcs", "firewall_rules", "dnszones"],
          "storage" => ["storage_pools", "buckets"],
        }
        # "the movie star/and the rest"
        groupings["services"] = MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] } - groupings.values.flatten
      elsif @group_by == :omnibus
        prefix = "mu" if prefix.empty? # so that appnames aren't ever empty
      end

      # Find any previous deploys with this particular profile, which we'll use
      # later for --diff.
      @existing_deploys = {}
      @existing_deploys_by_id = {}
      @origins = {}
      @types_found_in = {}
      groupings.each_pair { |appname, types|
        allowed_types = @types.map { |t| MU::Cloud.resource_types[t][:cfg_plural] }
        next if (types & allowed_types).size == 0
        origin = {
          "appname" => prefix+appname,
          "types" => (types & allowed_types).sort,
          "habitats" => @habitats.sort,
          "group_by" => @group_by.to_s
        }

        @existing_deploys[appname] = MU::MommaCat.findMatchingDeploy(origin)
        if @existing_deploys[appname]
          @existing_deploys_by_id[@existing_deploys[appname].deploy_id] = @existing_deploys[appname]
          @origins[appname] = origin
          origin['types'].each { |t|
            @types_found_in[t] = @existing_deploys[appname]
          }
        end
      }

      groupings.each_pair { |appname, types|
        allowed_types = @types.map { |t| MU::Cloud.resource_types[t][:cfg_plural] }
        next if (types & allowed_types).size == 0

        bok = { "appname" => prefix+appname }
        if @scrub_mu_isms
          bok["scrub_mu_isms"] = true
        end
        if @target_creds
          bok["credentials"] = @target_creds
        end

        count = 0
        if @diff
          if !@existing_deploys[appname]
            MU.log "--diff was set but I failed to find a deploy like '#{appname}' to compare to (have #{@existing_deploys.keys.join(", ")})", MU::ERR, details: @origins[appname]
            exit 1
          else
            MU.log "Will diff current live resources against #{@existing_deploys[appname].deploy_id}", MU::NOTICE, details: @origins[appname]
          end
        end

        threads = []
        timers = {}
        walltimers = {}
        @clouds.each { |cloud|
          @scraped.each_pair { |type, resources|
            typestart = Time.now
            res_class = begin
              MU::Cloud.resourceClass(cloud, type)
            rescue MU::Cloud::MuCloudResourceNotImplemented
              # XXX I don't think this can actually happen
              next
            end
            next if !types.include?(res_class.cfg_plural)

            bok[res_class.cfg_plural] ||= []
            timers[type] ||= {}

            class_semaphore = Mutex.new

            Thread.abort_on_exception = true
            resources.values.each { |obj_thr|
              obj_desc = nil
              begin
                obj_desc = obj_thr.cloud_desc
              rescue StandardError
              ensure
                if !obj_desc
                  MU.log cloud+" "+type.to_s+" "+obj_thr.cloud_id+" did not return a cloud descriptor, skipping", MU::WARN
                  next
                end
              end
              threads << Thread.new(obj_thr) { |obj|
                start = Time.now

                kitten_cfg = obj.toKitten(rootparent: @default_parent, billing: @billing, habitats: @habitats, types: @types)
                if kitten_cfg and (!@pattern or @pattern.match(kitten_cfg['name']))
                  print "."
                  kitten_cfg.delete("credentials") if @target_creds
                  class_semaphore.synchronize {
                    bok[res_class.cfg_plural] << kitten_cfg
                    if !kitten_cfg['cloud_id']
                      MU.log "No cloud id in this #{res_class.cfg_name} kitten!", MU::ERR, details: kitten_cfg
                    end
                    timers[type][kitten_cfg['cloud_id']] = (Time.now - start)
                  }
                  count += 1
                end
              }

            }

            threads.each { |t|
              t.join
            }

            puts ""
            bok[res_class.cfg_plural].sort! { |a, b|
              strs = [a, b].map { |x|
                if x['cloud_id']
                  x['cloud_id']
                elsif x['parent'] and ['parent'].respond_to?(:id) and kitten_cfg['parent'].id
                  x['name']+x['parent'].id
                elsif x['project']
                  x['name']+x['project']
                else
                  x['name']
                end
              }
              strs[0] <=> strs[1]
            }

            # If we've got duplicate names in here, try to deal with it
            bok[res_class.cfg_plural].each { |kitten_cfg|
              bok[res_class.cfg_plural].each { |sibling|
                next if kitten_cfg == sibling
                if sibling['name'] == kitten_cfg['name']
                  MU::Adoption.deDuplicateName(kitten_cfg, res_class)
                  MU.log "De-duplication: Renamed #{res_class.cfg_name} name '#{sibling['name']}' => '#{kitten_cfg['name']}'", MU::NOTICE
                  break
                end
              }
            }
            walltimers[type] ||= 0
            walltimers[type] += (Time.now - typestart)
          }
        }

        timers.each_pair { |type, resources|
          next if resources.empty?
          total = resources.values.sum
          top_5 =  resources.keys.sort { |a, b|
            resources[b] <=> resources[a]
          }.slice(0, 5).map { |k|
            k.to_s+": "+sprintf("%.2fs", resources[k])
          }
          if walltimers[type] < 45
            MU.log "Kittened #{resources.size.to_s} eligible #{type}s in #{sprintf("%.2fs", walltimers[type])}"
          else
            MU.log "Kittened #{resources.size.to_s} eligible #{type}s in #{sprintf("%.2fs", walltimers[type])} (CPU time #{sprintf("%.2fs", total)}, avg #{sprintf("%.2fs", total/resources.size)}). Top 5:", MU::NOTICE, details: top_5
          end
        }

        # No matching resources isn't necessarily an error
        next if count == 0 or bok.nil?

# Now walk through all of the Refs in these objects, resolve them, and minimize
# their config footprint
        MU.log "Minimizing footprint of #{count.to_s} found resources", MU::DEBUG

        generated_deploy = generateStubDeploy(bok)
        @boks[bok['appname']] = vacuum(bok, origin: @origins[appname], deploy: generated_deploy, save: @savedeploys)

        if @diff and !@existing_deploys[appname]
          MU.log "diff flag set, but no comparable deploy provided for #{bok['appname']}", MU::ERR
          exit 1
        end

        if @diff
          prev_vacuumed = vacuum(@existing_deploys[appname].original_config, deploy: @existing_deploys[appname], keep_missing: true, copy_from: generated_deploy)
          prevcfg = MU::Config.manxify(prev_vacuumed)
          if !prevcfg
            MU.log "#{@existing_deploys[appname].deploy_id} didn't have a working original config for me to compare", MU::ERR
            exit 1
          end
          newcfg = MU::Config.manxify(@boks[bok['appname']])
          report = prevcfg.diff(newcfg)

          if report

            if MU.muCfg['adopt_change_notify']
              notifyChanges(@existing_deploys[appname], report.freeze)
            end
            if @merge
              MU.log "Saving changes to #{@existing_deploys[appname].deploy_id}"
              @existing_deploys[appname].updateBasketofKittens(newcfg, save_now: true)
            end
          end

        end
      }
      @boks
    end
scrapeClouds() click to toggle source

Walk cloud providers with available credentials to discover resources

# File modules/mu/adoption.rb, line 55
    def scrapeClouds()
      @default_parent = nil

      @clouds.each { |cloud|
        cloudclass = MU::Cloud.cloudClass(cloud)
        next if cloudclass.listCredentials.nil?

        if cloud == "Google" and !@parent and @target_creds
          dest_org = MU::Cloud::Google.getOrg(@target_creds)
          if dest_org
            @default_parent = dest_org.name
          end
        end

        cloudclass.listCredentials.each { |credset|
          next if @sources and !@sources.include?(credset)
          cfg = cloudclass.credConfig(credset)
          if cfg and cfg['restrict_to_habitats']
            cfg['restrict_to_habitats'] << cfg['project'] if cfg['project']
          end

          if @parent
# TODO handle different inputs (cloud_id, etc)
# TODO do something about vague matches
            found = MU::MommaCat.findStray(
              cloud,
              "folders",
              flags: { "display_name" => @parent },
              credentials: credset,
              allow_multi: false,
              dummy_ok: true,
              debug: false
            )
            if found and found.size == 1
              @default_parent = found.first
            end
          end

          @types.each { |type|
            begin
              resclass = MU::Cloud.resourceClass(cloud, type)
            rescue ::MU::Cloud::MuCloudResourceNotImplemented
              next
            end
            if !resclass.instance_methods.include?(:toKitten)
              MU.log "Skipping MU::Cloud::#{cloud}::#{type} (resource has not implemented #toKitten)", MU::WARN
              next
            end
            MU.log "Scraping #{cloud}/#{credset} for #{resclass.cfg_plural}"

            found = MU::MommaCat.findStray(
              cloud,
              type,
              credentials: credset,
              allow_multi: true,
              habitats: @habitats.dup,
              region: @regions,
              dummy_ok: true,
              skip_provider_owned: true,
#              debug: false#,
            )


            if found and found.size > 0
              if resclass.cfg_plural == "habitats"
                found.reject! { |h|
                  !cloudclass.listHabitats(credset).include?(h.cloud_id)
                }
              end
              MU.log "Found #{found.size.to_s} raw #{resclass.cfg_plural} in #{cloud}"
              @scraped[type] ||= {}
              found.each { |obj|
                if obj.habitat and !cloudclass.listHabitats(credset).include?(obj.habitat)
                  next
                end

                # XXX apply any filters (e.g. MU-ID tags)
                if obj.cloud_id.nil?
                  MU.log "This damn thing gave me no cloud id, what do I even do with that", MU::ERR, details: obj
                  exit
                end
                @scraped[type][obj.cloud_id] = obj
              }
            end

          }
        }
      }

      if @parent and !@default_parent
        MU.log "Failed to locate a folder that resembles #{@parent}", MU::ERR
      end
      MU.log "Scraping complete"

      @scraped
    end

Private Instance Methods

catalogResources() click to toggle source

Go through everything we've scraped and update our mappings of cloud ids and bare name fields, so that resources can reference one another portably by name.

# File modules/mu/adoption.rb, line 956
def catalogResources
end
crawlChangeReport(tier, parent_key = nil, indent: "") click to toggle source

@param tier [Hash] @param parent_key [String]

# File modules/mu/adoption.rb, line 408
def crawlChangeReport(tier, parent_key = nil, indent: "")
  report = []
  if tier.is_a?(Array)
    tier.each { |a|
      sub_report = crawlChangeReport(a, parent_key)
      report.concat(sub_report) if sub_report and !sub_report.empty?
    }
  elsif tier.is_a?(Hash)
    if tier[:action]
      preposition = if tier[:action] == :added
        "to"
      elsif tier[:action] == :removed
        "from"
      else
        "in"
      end

      name = ""
      type_of = parent_key.sub(/s$|\[.*/, '') if parent_key
      loc = tier[:habitat]

      if tier[:value] and tier[:value].is_a?(Hash)
        name, loc = MU::MommaCat.getChunkName(tier[:value], type_of)
      elsif parent_key
        name = parent_key
      end

      path_str = []
      slack_path_str = ""
      if tier[:parents] and tier[:parents].size > 2
        path = tier[:parents].clone
        slack_path_str += "#{preposition} \*"+path.join(" ⇨ ")+"\*" if path.size > 0
        path.shift
        path.shift
        path.pop if path.last == name
        for c in (0..(path.size-1)) do
          path_str << ("  " * (c+2)) + (path[c] || "<nil>")
        end
      end
      path_str << "" if !path_str.empty?

      plain = (name ? name : type_of) if name or type_of
      plain ||= "" # XXX but this is a problem
      slack = "`"+plain+"`"

      plain += " ("+loc+")" if loc and !loc.empty?
      color = plain

      if tier[:action] == :added
        color = "+ ".green + plain
        plain = "+ " + plain
        slack += " added"
      elsif tier[:action] == :removed
        color = "- ".red + plain
        plain = "- " + plain
        slack += " removed"
      end

      slack += " #{tier[:action]} #{preposition} \*#{loc}\*" if loc and !loc.empty? and [Array, Hash].include?(tier[:value].class)

      plain = path_str.join(" => \n") + indent + plain
      color = path_str.join(" => \n") + indent + color

      slack += " "+slack_path_str if !slack_path_str.empty?
      myreport = {
        "slack" => slack,
        "plain" => plain,
        "color" => color
      }

      append = ""
      if tier[:value] and (tier[:value].is_a?(Array) or tier[:value].is_a?(Hash))
        if tier[:value].is_a?(Hash)
          if name
            tier[:value].delete("entity")
            tier[:value].delete(name.sub(/\[.*/, '')) if name
          end
          if (tier[:value].keys - ["id", "name", "type"]).size > 0
            myreport["details"] = tier[:value].clone
            append = PP.pp(tier[:value], '').gsub(/(^|\n)/, '\1'+indent)
          end
        else
          append = indent+"["+tier[:value].map { |v| MU::MommaCat.getChunkName(v, type_of).reverse.join("/") || v.to_s.light_blue }.join(", ")+"]"
          slack += " #{tier[:action].to_s}: "+tier[:value].map { |v| MU::MommaCat.getChunkName(v, type_of).reverse.join("/") || v.to_s }.join(", ")
        end
      else
        tier[:value] ||= "<nil>"
        if ![:removed].include?(tier[:action])
          myreport["slack"] += ". New #{tier[:field] ? "`"+tier[:field]+"`" : :value}: \*#{tier[:value]}\*"
        else
          myreport["slack"] += " (was \*#{tier[:value]}\*)"
        end
        append = tier[:value].to_s.bold
      end

      if append and !append.empty?
        myreport["plain"] += " =>\n  "+indent+append
        myreport["color"] += " =>\n  "+indent+append
      end

      report << myreport if tier[:action]
    end

    # Just because we've got changes at this level doesn't mean there aren't
    # more further down.
    tier.each_pair { |k, v|
      next if !(v.is_a?(Hash) or v.is_a?(Array))
      sub_report = crawlChangeReport(v, k, indent: indent+"  ")
      report.concat(sub_report) if sub_report and !sub_report.empty?
    }
  end

  report
end
generateStubDeploy(bok) click to toggle source

@return [MU::MommaCat]

# File modules/mu/adoption.rb, line 871
    def generateStubDeploy(bok)
#      hashify Ref objects before passing into here... or do we...?

      time = Time.new
      timestamp = time.strftime("%Y%m%d%H").to_s;
      timestamp.freeze

      retries = 0
      deploy_id = nil
      seed = nil
      begin
        raise MuError, "Failed to allocate an unused MU-ID after #{retries} tries!" if retries > 70
        seedsize = 1 + (retries/10).abs
        seed = (0...seedsize+1).map { ('a'..'z').to_a[rand(26)] }.join
        deploy_id = bok['appname'].upcase + "-ADOPT-" + timestamp + "-" + seed.upcase
      end while MU::MommaCat.deploy_exists?(deploy_id) or seed == "mu" or seed[0] == seed[1]

      MU.setVar("deploy_id", deploy_id)
      MU.setVar("appname", bok['appname'].upcase)
      MU.setVar("environment", "ADOPT")
      MU.setVar("timestamp", timestamp)
      MU.setVar("seed", seed)
      MU.setVar("handle", MU::MommaCat.generateHandle(seed))

      deploy = MU::MommaCat.new(
        deploy_id,
        create: true,
        config: bok,
        environment: "adopt",
        appname: bok['appname'].upcase,
        timestamp: timestamp,
        nocleanup: true,
        no_artifacts: !(@savedeploys),
        set_context_to_me: true,
        mu_user: MU.mu_user
      )

      MU::Cloud.resource_types.each_pair { |typename, attrs|
        if bok[attrs[:cfg_plural]]
          bok[attrs[:cfg_plural]].each { |kitten|

            if !@scraped[typename][kitten['cloud_id']]
              MU.log "No object in scraped tree for #{attrs[:cfg_name]} #{kitten['cloud_id']} (#{kitten['name']})", MU::ERR, details: kitten
              if kitten['cloud_id'].nil?
                pp caller
                exit
              end
              next
            end

            MU.log "Inserting #{attrs[:cfg_name]} #{kitten['name']} (#{kitten['cloud_id']}) into stub deploy", MU::DEBUG, details: @scraped[typename][kitten['cloud_id']]

            @scraped[typename][kitten['cloud_id']].config!(kitten)

            deploy.addKitten(
              attrs[:cfg_plural],
              kitten['name'],
              @scraped[typename][kitten['cloud_id']],
              do_notify: true
            )
          }
        end
      }

      deploy
    end
notifyChanges(deploy, report) click to toggle source
# File modules/mu/adoption.rb, line 524
def notifyChanges(deploy, report)
  snippet_threshold = (MU.muCfg['adopt_change_notify'] && MU.muCfg['adopt_change_notify']['slack_snippet_threshold']) || 5

  report.each_pair { |res_type, resources|
    shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(res_type, false)
    next if !shortclass # we don't really care about Mu metadata changes
    resources.each_pair { |name, data|
      if MU::MommaCat.getChunkName(data[:value], res_type).first.nil?
        symbol = if data[:action] == :added
          "+".green
        elsif data[:action] == :removed
          "-".red
        else
          "~".yellow
        end
        puts (symbol+" "+res_type+"["+name+"]")
      end

      noun = shortclass ? shortclass.to_s : res_type.capitalize
      verb = if data[:action]
        data[:action].to_s
      else
        "modified"
      end

      changes = crawlChangeReport(data.freeze, res_type)

      slacktext = "#{noun} \*#{name}\* was #{verb}"
      if data[:habitat]
        slacktext += " in \*#{data[:habitat]}\*"
      end
      snippets = []

      if [:added, :removed].include?(data[:action]) and data[:value]
        snippets << { text: "```"+JSON.pretty_generate(data[:value])+"```" }
      else
        changes.each { |c|
          slacktext += "\n • "+c["slack"]
          if c["details"]
            details = JSON.pretty_generate(c["details"])
            snippets << { text: "```"+JSON.pretty_generate(c["details"])+"```" }
          end
        }
      end

      changes.each { |c|
        puts c["color"]
      }
      puts ""

      if MU.muCfg['adopt_change_notify'] and MU.muCfg['adopt_change_notify']['slack']
        deploy.sendAdminSlack(slacktext, scrub_mu_isms: MU.muCfg['adopt_scrub_mu_isms'], snippets: snippets, noop: false)
      end

    }
  }

end
resolveReferences(cfg, deploy, parent) click to toggle source
# File modules/mu/adoption.rb, line 743
def resolveReferences(cfg, deploy, parent)
  mask_deploy_id = false

  check_deploy_id = Proc.new { |cfgblob|
    (deploy and
     (cfgblob.is_a?(MU::Config::Ref) or cfgblob.is_a?(Hash)) and
     cfgblob['deploy_id'] and
     cfgblob['deploy_id'] != deploy.deploy_id and
     @diff and
     @types_found_in[cfgblob['type']] and
     @types_found_in[cfgblob['type']].deploy_id == cfgblob['deploy_id']
    )
  }

  mask_deploy_id = check_deploy_id.call(cfg)

  if cfg.is_a?(MU::Config::Ref)
    if mask_deploy_id
      cfg.delete("deploy_id")
      cfg.delete("mommacat")
      cfg.kitten(deploy)
    else
      cfg.kitten(deploy) || cfg.kitten
    end

    hashcfg = cfg.to_h

    if cfg.kitten
      littermate = deploy.findLitterMate(type: cfg.type, name: cfg.name, cloud_id: cfg.id, habitat: cfg.habitat)

      if littermate and littermate.config['name']
        hashcfg['name'] = littermate.config['name']
        hashcfg.delete("id") if hashcfg["name"]
        hashcfg
      elsif cfg.deploy_id and cfg.name and @savedeploys
        hashcfg.delete("id") if hashcfg["name"]
        hashcfg
      elsif cfg.id
        littermate = deploy.findLitterMate(type: cfg.type, cloud_id: cfg.id, habitat: cfg.habitat)
        if littermate and littermate.config['name']
          hashcfg['name'] = littermate.config['name']
          hashcfg.delete("id") if hashcfg["name"]
        elsif !@savedeploys
          hashcfg.delete("deploy_id")
          hashcfg.delete("name")
        else
          hashcfg.delete("name") if cfg.id and !cfg.deploy_id
        end
      end
    elsif hashcfg["id"] and !hashcfg["name"]
      hashcfg.delete("deploy_id")
    else
      raise Incomplete.new "Failed to resolve reference on behalf of #{parent}", details: hashcfg
    end
    hashcfg.delete("deploy_id") if hashcfg['deploy_id'] == deploy.deploy_id

    if parent and parent.config
      cred_cfg = MU::Cloud.cloudClass(parent.cloud).credConfig(parent.credentials)

      if parent.config['region'] == hashcfg['region'] or
         cred_cfg['region'] == hashcfg['region']
        hashcfg.delete("region")
      end

      habitat_id = if cfg.habitat
        if cfg.habitat.is_a?(MU::Config::Ref)
          cfg.habitat.id
        else
          cfg.habitat['id']
        end
      else
        nil
      end

      if habitat_id
        if (parent.config['habitat'] and parent.config['habitat']['id'] == habitat_id) or
           cred_cfg['account_number'] == habitat_id or # AWS
           cred_cfg['project'] == habitat_id or # GCP
           cred_cfg['subscription'] == habitat_id # Azure
          hashcfg.delete('habitat') 
        end
      end

      if parent.config['credentials'] == hashcfg['credentials']
        hashcfg.delete("credentials")
      end
    end
    cfg = hashcfg
  elsif cfg.is_a?(Hash)
    deletia = []
    cfg.each_pair { |key, value|
      begin
        cfg[key] = resolveReferences(value, deploy, parent)
      rescue Incomplete
        MU.log "Dropping unresolved key #{key}", MU::WARN, details: cfg
        deletia << key
      end
    }
    deletia.each { |key|
      cfg.delete(key)
    }
    cfg = nil if cfg.empty? and deletia.size > 0
  elsif cfg.is_a?(Array)
    new_array = []
    cfg.each { |value|
      begin
        new_item = resolveReferences(value, deploy, parent)
        if !new_item
          MU.log "Dropping unresolved value", MU::WARN, details: value
        else
          new_array << new_item
        end
      rescue Incomplete
        MU.log "Dropping unresolved value", MU::WARN, details: value
      end
    }
    cfg = new_array.uniq
  end

  if mask_deploy_id or check_deploy_id.call(cfg)
    cfg.delete("deploy_id")
    MU.log "#{parent} in #{deploy.deploy_id} references something in #{@types_found_in[cfg['type']].deploy_id}, ditching extraneous deploy_id", MU::DEBUG, details: cfg.to_h
  end

  cfg
end
scrubSchemaDefaults(conf_chunk, schema_chunk, depth = 0, type: nil) click to toggle source
# File modules/mu/adoption.rb, line 583
def scrubSchemaDefaults(conf_chunk, schema_chunk, depth = 0, type: nil)
  return if schema_chunk.nil?

  if !conf_chunk.nil? and schema_chunk["properties"].kind_of?(Hash) and conf_chunk.is_a?(Hash)
    deletia = []
    schema_chunk["properties"].each_pair { |key, subschema|
      next if !conf_chunk[key]
      shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(key, false)

      if subschema["default_if"]
        subschema["default_if"].each { |cond|
          if conf_chunk[cond["key_is"]] == cond["value_is"]
            subschema["default"] = cond["set"]
            break
          end
        }
      end

      if subschema["default"] and conf_chunk[key] == subschema["default"]
        deletia << key
      elsif ["array", "object"].include?(subschema["type"])
        scrubSchemaDefaults(conf_chunk[key], subschema, depth+1, type: shortclass)
      end
    }
    deletia.each { |key| conf_chunk.delete(key) }
  elsif schema_chunk["type"] == "array" and conf_chunk.kind_of?(Array)
    conf_chunk.each { |item|
      # this bit only happens at the top-level key for a resource type, in
      # theory
      realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"] and MU::Cloud.supportedClouds.include?(item['cloud'])

        _toplevel_required, cloudschema = MU::Cloud.resourceClass(item['cloud'], type).schema(self)

        newschema = schema_chunk["items"].dup
        newschema["properties"].merge!(cloudschema)
        newschema
      else
        schema_chunk["items"].dup
      end
      next if ["array", "object"].include?(realschema["type"])

      scrubSchemaDefaults(item, realschema, depth+1, type: type)
    }
  end

  conf_chunk
end
vacuum(bok, origin: nil, save: false, deploy: nil, copy_from: nil, keep_missing: false) click to toggle source

Recursively walk through a BoK hash, validate all {MU::Config::Ref} objects, convert them to hashes, and pare them down to the minimal representation (remove extraneous attributes that match the parent object). Do the same for our main objects: if they all use the same credentials, for example, remove the explicit credentials attributes and set that value globally, once.

# File modules/mu/adoption.rb, line 638
def vacuum(bok, origin: nil, save: false, deploy: nil, copy_from: nil, keep_missing: false)

  globals = {
    'cloud' => {},
    'credentials' => {},
    'region' => {},
    'billing_acct' => {},
    'us_only' => {},
  }
  MU::Cloud.resource_types.values.each { |attrs|
    if bok[attrs[:cfg_plural]]
      processed = []
      bok[attrs[:cfg_plural]].each { |resource|
        globals.each_pair { |field, counts|
          if resource[field]
            counts[resource[field]] ||= 0
            counts[resource[field]] += 1
          end
        }
        obj = deploy.findLitterMate(type: attrs[:cfg_plural], name: resource['name'])
        inject_metadata = save
        if obj.nil? and copy_from
          obj = copy_from.findLitterMate(type: attrs[:cfg_plural], name: resource['name'])
          if obj
            inject_metadata = true
            obj.intoDeploy(deploy, force: true)
          end
        end

        begin
          raise Incomplete if obj.nil?
          if inject_metadata
            deploydata = obj.notify
            deploy.notify(attrs[:cfg_plural], resource['name'], deploydata, triggering_node: obj)
          end
          new_cfg = resolveReferences(resource, deploy, obj)
          new_cfg.delete("cloud_id")
          cred_cfg = MU::Cloud.cloudClass(obj.cloud).credConfig(obj.credentials)
          if cred_cfg['region'] == new_cfg['region']
            new_cfg.delete('region')
          end
          if cred_cfg['default']
            new_cfg.delete('credentials')
            new_cfg.delete('habitat')
          end
          processed << new_cfg
        rescue Incomplete
          if keep_missing
            processed << resource
          else
            MU.log "#{attrs[:cfg_name]} #{resource['name']} didn't show up from findLitterMate", MU::WARN, details: deploy.original_config[attrs[:cfg_plural]].reject { |r| r['name'] != "" }
          end
        end
      }

      deploy.original_config[attrs[:cfg_plural]] = processed
      bok[attrs[:cfg_plural]] = processed
    end
  }

  # Pare out global values like +cloud+ or +region+ that appear to be
  # universal in the deploy we're creating.
  scrub_globals = Proc.new { |h, field|
    if h.is_a?(Hash)
      newhash = {}
      h.each_pair { |k, v|
        next if k == field
        newhash[k] = scrub_globals.call(v, field)
      }
      h = newhash
    elsif h.is_a?(Array)
      newarr = []
      h.each { |v|
        newarr << scrub_globals.call(v, field)
      }
      h = newarr.uniq
    end
    h
  }

  globals.each_pair { |field, counts|
    next if counts.size != 1
    bok[field] = counts.keys.first
    MU.log "Setting global default #{field} to #{bok[field]} (#{deploy.deploy_id})", MU::DEBUG
    MU::Cloud.resource_types.values.each { |attrs|
      if bok[attrs[:cfg_plural]]
        new_resources = []
        bok[attrs[:cfg_plural]].each { |resource|
          new_resources << scrub_globals.call(resource, field)
        }
        bok[attrs[:cfg_plural]] = new_resources
      end
    }
  }

  scrubSchemaDefaults(bok, MU::Config.schema)

  if save
    MU.log "Committing adopted deployment to #{MU.dataDir}/deployments/#{deploy.deploy_id}", MU::NOTICE, details: origin
    deploy.save!(force: true, origin: origin)
  end

  bok
end