class MU::Cleanup

Routines for removing cloud resources.

Constants

TYPES_IN_ORDER

Resource types, in the order in which we generally have to clean them up to disentangle them from one another.

Public Class Methods

run(deploy_id, noop: false, skipsnapshots: false, onlycloud: false, verbosity: MU::Logger::NORMAL, web: false, ignoremaster: false, skipcloud: false, mommacat: nil, credsets: nil, regions: nil, habitats: nil) click to toggle source

Purge all resources associated with a deployment. @param deploy_id [String]: The identifier of the deployment to remove (typically seen in the MU-ID tag on a resource). @param noop [Boolean]: Do not delete resources, merely list what would be deleted. @param skipsnapshots [Boolean]: Refrain from saving final snapshots of volumes and databases before deletion. @param onlycloud [Boolean]: Purge cloud resources, but skip purging all Mu master metadata, ssh keys, etc. @param verbosity [Integer]: Debug level for MU.log output @param web [Boolean]: Generate web-friendly output. @param ignoremaster [Boolean]: Ignore the tags indicating the originating MU master server when deleting. @param regions [Array<String>]: Operate only on these regions @param habitats [Array<String>]: Operate only on these accounts/projects/subscriptions @return [void]

# File modules/mu/cleanup.rb, line 48
    def self.run(deploy_id, noop: false, skipsnapshots: false, onlycloud: false, verbosity: MU::Logger::NORMAL, web: false, ignoremaster: false, skipcloud: false, mommacat: nil, credsets: nil, regions: nil, habitats: nil)
      MU.setLogging(verbosity, web)
      @noop = noop
      @skipsnapshots = skipsnapshots
      @onlycloud = onlycloud
      @skipcloud = skipcloud
      @ignoremaster = ignoremaster
      @deploy_id = deploy_id

      if @skipcloud and @onlycloud # you actually mean noop
        @onlycloud = @skipcloud = false
        @noop = true
      end

      MU.setVar("dataDir", (MU.mu_user == "mu" ? MU.mainDataDir : Etc.getpwnam(MU.mu_user).dir+"/.mu/var"))

      # Load up our deployment metadata
      if !mommacat.nil?
        @mommacat = mommacat
      else
        begin
          deploy_dir = File.expand_path("#{MU.dataDir}/deployments/"+deploy_id)
          if Dir.exist?(deploy_dir)
#                                               key = OpenSSL::PKey::RSA.new(File.read("#{deploy_dir}/public_key"))
#                                               deploy_secret = key.public_encrypt(File.read("#{deploy_dir}/deploy_secret"))
            FileUtils.touch("#{deploy_dir}/.cleanup") if !@noop
          else
            MU.log "I don't see a deploy named #{deploy_id}.", MU::WARN
            MU.log "Known deployments:\n#{Dir.entries(deploy_dir).reject { |item| item.match(/^\./) or !File.exist?(deploy_dir+"/"+item+"/public_key") }.join("\n")}", MU::WARN
            MU.log "Searching for remnants of #{deploy_id}, though this may be an invalid MU-ID.", MU::WARN
          end
          @mommacat = MU::MommaCat.new(deploy_id, mu_user: MU.mu_user, delay_descriptor_load: true)
        rescue StandardError => e
          MU.log "Can't load a deploy record for #{deploy_id} (#{e.inspect}), cleaning up resources by guesswork", MU::WARN, details: e.backtrace
          MU.setVar("deploy_id", deploy_id)
        end
      end

      @regionsused = @mommacat.regionsUsed if @mommacat
      @credsused = @mommacat.credsUsed if @mommacat
      @habitatsused = @mommacat.habitatsUsed if @mommacat

      if !@skipcloud
        creds = listUsedCredentials(credsets)

        cloudthreads = []

        had_failures = false

        creds.each_pair { |provider, credsets_outer|
          cloudthreads << Thread.new(provider, credsets_outer) { |cloud, credsets_inner|
            Thread.abort_on_exception = false
            cleanCloud(cloud, habitats, regions, credsets_inner)
          } # cloudthreads << Thread.new(provider, credsets) { |cloud, credsets_outer|
          cloudthreads.each do |t|
            t.join
          end
        } # creds.each_pair { |provider, credsets|


        # Knock habitats and folders, which would contain the above resources,
        # once they're all done.
        creds.each_pair { |provider, credsets_inner|
          credsets_inner.keys.each { |credset|
            next if @credsused and !@credsused.include?(credset)
            ["Habitat", "Folder"].each { |t|
              flags = {
                "onlycloud" => @onlycloud,
                "skipsnapshots" => @skipsnapshots
              }
              if !call_cleanup(t, credset, provider, flags, nil)
                had_failures = true
              end
            }
          }
        }

        creds.each_pair { |provider, credsets_inner|
          cloudclass = MU::Cloud.cloudClass(provider)
          credsets_inner.keys.each { |c|
            cloudclass.cleanDeploy(MU.deploy_id, credentials: c, noop: @noop)
          }
        }
      end

      # Scrub any residual Chef records with matching tags
      if !@onlycloud and (@mommacat.nil? or @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0) and !@noop
        MU.supportedGroomers.each { |g|
          groomer = MU::Groomer.loadGroomer(g)
          groomer.cleanup(MU.deploy_id, @noop)
        }
      end

      if had_failures
        MU.log "Had cleanup failures, exiting", MU::ERR
        File.unlink("#{deploy_dir}/.cleanup") if !@noop
        exit 1
      end

      if !@onlycloud and !@noop and @mommacat
        @mommacat.purge!
      end

      if !@onlycloud
        MU::Master.purgeDeployFromSSH(MU.deploy_id, noop: @noop)
      end

      if !@noop and !@skipcloud and (@mommacat.nil? or @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0)
#        MU::Master.syncMonitoringConfig
      end

    end

Private Class Methods

call_cleanup(type, credset, provider, flags, region) click to toggle source

Wrapper for dynamically invoking resource type cleanup methods. @param type [String]: @param credset [String]: @param provider [String]: @param flags [Hash]: @param region [String]:

# File modules/mu/cleanup.rb, line 344
def self.call_cleanup(type, credset, provider, flags, region)
  Thread.current.thread_variable_set("type", type)
  if @mommacat.nil? or @mommacat.numKittens(types: [type]) > 0
    if @mommacat

      found = @mommacat.findLitterMate(type: type, return_all: true, credentials: credset)

      if found
        flags['known'] = if found.is_a?(Array)
          found.map { |k| k.cloud_id }
        elsif found.is_a?(Hash)
          found.each_value.map { |k| k.cloud_id }
        else
          [found.cloud_id]
        end
      end
    end

    MU::Cloud.loadBaseType(type).cleanup(
      noop: @noop,
      ignoremaster: @ignoremaster,
      region: region,
      cloud: provider,
      flags: flags,
      credentials: credset,
      deploy_id: @deploy_id
    )
  else
    true
  end
end
cleanCloud(cloud, habitats, regions, credsets) click to toggle source
# File modules/mu/cleanup.rb, line 184
def self.cleanCloud(cloud, habitats, regions, credsets)
  cloudclass = MU::Cloud.cloudClass(cloud)
  credsets.each_pair { |credset, acct_regions|
    next if @credsused and !@credsused.include?(credset)
    global_vs_region_semaphore = Mutex.new
    global_done = {}
    regionthreads = []
    acct_regions.each { |r|
      if @regionsused
        if @regionsused.size > 0
          next if !@regionsused.include?(r)
        else
          next if r != cloudclass.myRegion(credset)
        end
      end
      if regions and !regions.empty?
        next if !regions.include?(r)
        MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{r}...", MU::NOTICE
      end
      regionthreads << Thread.new {
        Thread.abort_on_exception = false
        MU.setVar("curRegion", r)
        cleanRegion(cloud, credset, r, global_vs_region_semaphore, global_done, habitats)
      } # regionthreads << Thread.new {
    } # acct_regions.each { |r|
    regionthreads.each do |t|
      t.join
    end
  }
end
cleanHabitat(cloud, credset, region, habitat, global_vs_region_semaphore, global_done) click to toggle source
# File modules/mu/cleanup.rb, line 289
def self.cleanHabitat(cloud, credset, region, habitat, global_vs_region_semaphore, global_done)
  had_failures = false
  if habitat != ""
    MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{region}, habitat #{habitat}", MU::NOTICE
  end

  flags = {
    "habitat" => habitat,
    "onlycloud" => @onlycloud,
    "skipsnapshots" => @skipsnapshots,
  }
  TYPES_IN_ORDER.each { |t|
    begin
      skipme = false
      global_vs_region_semaphore.synchronize {
        if MU::Cloud.resourceClass(cloud, t).isGlobal?
          global_done[habitat] ||= []
          if !global_done[habitat].include?(t)
            global_done[habitat] << t
            flags['global'] = true
          else
            skipme = true
          end
        end
      }
      next if skipme
    rescue MU::Cloud::MuDefunctHabitat, MU::Cloud::MuCloudResourceNotImplemented
      next
    rescue MU::MuError, NoMethodError => e
      MU.log "While checking mu/providers/#{cloud.downcase}/#{cloudclass.cfg_name} for global-ness in cleanup: "+e.message, MU::WARN
      next
    rescue ::Aws::EC2::Errors::AuthFailure, ::Google::Apis::ClientError => e
      MU.log e.message+" in "+region, MU::ERR
      next
    end

    begin
      if !call_cleanup(t, credset, cloud, flags, region)
        had_failures = true
      end
    rescue MU::Cloud::MuDefunctHabitat, MU::Cloud::MuCloudResourceNotImplemented
      next
    end
  }

  had_failures
end
cleanRegion(cloud, credset, region, global_vs_region_semaphore, global_done, habitats) click to toggle source
# File modules/mu/cleanup.rb, line 216
    def self.cleanRegion(cloud, credset, region, global_vs_region_semaphore, global_done, habitats)
      had_failures = false
      cloudclass = MU::Cloud.cloudClass(cloud)
      habitatclass = MU::Cloud.resourceClass(cloud, "Habitat")

      if !habitats
        habitats = []
        if $MU_CFG and $MU_CFG[cloud.downcase] and
           $MU_CFG[cloud.downcase][credset] and
           $MU_CFG[cloud.downcase][credset]["project"]
# XXX GCP credential schema needs an array for projects
          habitats << $MU_CFG[cloud.downcase][credset]["project"]
        end
        begin
          habitats.concat(cloudclass.listHabitats(credset, use_cache: false))
        rescue NoMethodError
        end
      end

      if habitats == []
        habitats << "" # dummy
        MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{region}", MU::NOTICE
      end
      habitats.uniq!

      # We do these in an order that unrolls dependent resources
      # sensibly, and we hit :Collection twice because AWS
      # CloudFormation sometimes fails internally.
      habitat_threads = []
      habitats.each { |habitat|
        if habitats and !habitats.empty? and habitat != ""
          next if !habitats.include?(habitat)
        end
        if @habitatsused and !@habitatsused.empty? and habitat != ""
          next if !@habitatsused.include?(habitat)
        end
        next if !habitatclass.isLive?(habitat, credset)

        habitat_threads << Thread.new {
          Thread.current.thread_variable_set("name", "#{cloud}/#{credset}/#{habitat}/#{region}")
          Thread.abort_on_exception = false
          if !cleanHabitat(cloud, credset, region, habitat, global_vs_region_semaphore, global_done)
            had_failures = true
          end
        } # TYPES_IN_ORDER.each { |t|
      } # habitats.each { |habitat|

      last_checkin = Time.now
      begin
        deletia = []
        habitat_threads.each { |t|
          if !t.status
            t.join
            deletia << t
          end
        }
        deletia.each { |t|
          habitat_threads.delete(t)
        }
        if (Time.now - last_checkin) > 120
          list = habitat_threads.map { |t|
            t.thread_variable_get("name") + (t.thread_variable_get("type") ? "/"+t.thread_variable_get("type") : "")
          }
          MU.log "Waiting on #{habitat_threads.size.to_s} habitat#{habitat_threads.size > 1 ? "s" : ""} in region #{region}", MU::NOTICE, details: list
          last_checkin = Time.now
        end
        sleep 10 if !habitat_threads.empty?
      end while !habitat_threads.empty?

      had_failures
    end
listUsedCredentials(credsets) click to toggle source
# File modules/mu/cleanup.rb, line 161
def self.listUsedCredentials(credsets)
  creds = {}
  MU::Cloud.availableClouds.each { |cloud|
    cloudclass = MU::Cloud.cloudClass(cloud)
    if $MU_CFG[cloud.downcase] and $MU_CFG[cloud.downcase].size > 0
      creds[cloud] ||= {}
      cloudclass.listCredentials.each { |credset|
        next if credsets and credsets.size > 0 and !credsets.include?(credset)
        next if @credsused and @credsused.size > 0 and !@credsused.include?(credset)
        MU.log "Will scan #{cloud} with credentials #{credset}"
        creds[cloud][credset] = cloudclass.listRegions(credentials: credset)
      }
    else
      if cloudclass.hosted?
        creds[cloud] ||= {}
        creds[cloud]["#default"] = cloudclass.listRegions
      end
    end
  }
  creds
end