class MU::Cloud::AWS::StoragePool

A storage pool as configured in {MU::Config::BasketofKittens::storage_pools}

Public Class Methods

cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) click to toggle source

Called by {MU::Cleanup}. Locates resources that were created by the currently-loaded deployment, and purges them. @param noop [Boolean]: If true, will only print what would be done @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server @param region [String]: The cloud provider region in which to operate @return [void]

# File modules/mu/providers/aws/storage_pool.rb, line 344
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
  MU.log "AWS::StoragePool.cleanup: need to support flags['known']", MU::DEBUG, details: flags

  supported_regions = %w{us-west-2 us-east-1 eu-west-1}
  if supported_regions.include?(region)
    begin 
      resp = MU::Cloud::AWS.efs(credentials: credentials, region: region).describe_file_systems
      return if resp.nil? or resp.file_systems.nil?
      storage_pools = resp.file_systems
    rescue Aws::EFS::Errors::AccessDeniedException
      MU.log "Storage Pools not supported in this account", MU::NOTICE
      return nil
    end

    our_pools = []

    if !storage_pools.empty?
      storage_pools.each{ |pool|
        tags = MU::Cloud::AWS.efs(credentials: credentials, region: region).describe_tags(
          file_system_id: pool.file_system_id
        ).tags

        found_muid = false
        found_master = false
        tags.each { |tag|
          found_muid = true if tag.key == "MU-ID" && tag.value == deploy_id
          found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip
        }
        next if !found_muid

        if ignoremaster
          our_pools << pool if found_muid
        else
          our_pools << pool if found_muid && found_master
        end
      }
    end

    # How to identify mount points in a reliable way? Mount points are not tagged, which means we can only reliably identify mount points based on a filesystem ID. We can you our deployment metadata, but it isn’t necessarily reliable
    # begin
      # resp = MU::Cloud::AWS.efs(credentials: credentials, region: region).delete_mount_target(
        # mount_target_id: "MountTargetId"
      # )
      # MU.log "Deleted mount target"
    # rescue Aws::EFS::Errors::BadRequest => e
      # MU.log "Mount target already deleted", MU::NOTICE if e.to_s.start_with?("invalid mount target ID")
    # end

    if !our_pools.empty?
      our_pools.each{ |pool|
        mount_targets = MU::Cloud::AWS.efs(credentials: credentials, region: region).describe_mount_targets(
          file_system_id: pool.file_system_id
        ).mount_targets

        if !mount_targets.empty?
          mount_targets.each{ |mp|
            MU.log "Deleting mount target #{mp.mount_target_id} for filesystem #{pool.name}: #{pool.file_system_id}"
            unless noop
              begin
                resp = MU::Cloud::AWS.efs(credentials: credentials, region: region).delete_mount_target(
                  mount_target_id: mp.mount_target_id
                  )
              rescue Aws::EFS::Errors::BadRequest => e
                MU.log "Mount target #{mp.mount_target_id} already deleted", MU::NOTICE if e.to_s.start_with?("invalid mount target ID")
              end
            end
          }
        end

        MU.log "Deleting filesystem #{pool.name}: #{pool.file_system_id}"
        unless noop
          attempts = 0
          begin
            resp = MU::Cloud::AWS.efs(credentials: credentials, region: region).delete_file_system(
              file_system_id: pool.file_system_id
            )
          rescue Aws::EFS::Errors::BadRequest => e
            MU.log "Filesystem #{pool.name}: #{pool.file_system_id} already deleted", MU::NOTICE if e.to_s.start_with?("invalid file system ID")
          rescue Aws::EFS::Errors::FileSystemInUse
            MU.log "Filesystem #{pool.name}: #{pool.file_system_id} is still in use, retrying", MU::NOTICE
            sleep 10
            attempts += 1
            raise MuError, "Failed to delete filesystem #{pool.name}: #{pool.file_system_id}, still in use." if attempts >= 6
            retry
          end
        end
      }
    end
  end
end
create_mount_target(cloud_id: nil, ip_address: nil, subnet_id: nil, security_groups: [], region: MU.curRegion, credentials: nil) click to toggle source

Create a mount point for an existing storage pool and attach it to a given subnet @param cloud_id [String]: The cloud provider's identifier of the storage pool. @param ip_address [String]: A private IP address that will be associated with mount point's network interface @param subnet_id [String]: The subnet_id to associate the mount point with @param security_groups [Array]: A list of security groups to associate with the mount point. @param region [String]: The cloud provider region

# File modules/mu/providers/aws/storage_pool.rb, line 197
def self.create_mount_target(cloud_id: nil, ip_address: nil, subnet_id: nil, security_groups: [], region: MU.curRegion, credentials: nil)
  MU.log "Creating mount target for filesystem #{cloud_id}"

  resp = MU::Cloud::AWS.efs(region: region, credentials: credentials).create_mount_target(
    file_system_id: cloud_id,
    subnet_id: subnet_id,
    ip_address: ip_address,
    security_groups: security_groups
  )

  attempts = 0
  retries = 0
  loop do
    MU.log "Waiting for #{resp.mount_target_id} to become available", MU::NOTICE if attempts % 10 == 0
    begin
      mount_target = MU::Cloud::AWS.efs(region: region, credentials: credentials).describe_mount_targets(
        mount_target_id: resp.mount_target_id 
      ).mount_targets.first
    rescue Aws::EFS::Errors::MountTargetNotFound
      if retries <= 3
        sleep 10
        retry
      else
        return nil
      end
    end

    break if mount_target.life_cycle_state == "available"
    raise MuError, "Failed to create mount target #{resp.mount_target_id }" if %w{deleting deleted}.include? mount_target.life_cycle_state
    sleep 10
    attempts += 1
    raise MuError, "timed out waiting for #{resp.mount_target_id }" if attempts >= 40
  end

  return resp
end
find(**args) click to toggle source

Locate an existing storage pool and return an array containing matching AWS resource descriptors for those that match. @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching storage pool

# File modules/mu/providers/aws/storage_pool.rb, line 108
def self.find(**args)
  found = {}

  if args[:cloud_id]
    resp = MU::Cloud::AWS.efs(region: args[:region], credentials: args[:credentials]).describe_file_systems(
      file_system_id: args[:cloud_id]
    ).file_systems.first
    found[args[:cloud_id]] = resp if resp
  elsif args[:tag_value]
    storage_pools = MU::Cloud::AWS.efs(region: args[:region], credentials: args[:credentials]).describe_file_systems.file_systems
  
    if !storage_pools.empty?
      storage_pools.each{ |pool|
        tags = MU::Cloud::AWS.efs(region: args[:region], credentials: args[:credentials]).describe_tags(
          file_system_id: pool.file_system_id
        ).tags

        value = nil
        tags.each{ |tag|
          if tag.key == args[:tag_key]
            value = tag.value
            break
          end
        }
        
        if value == args[:tag_value]
          found[pool.file_system_id] = pool
          break
        end
      }
    end
  else
    resp = MU::Cloud::AWS.efs(region: args[:region], credentials: args[:credentials]).describe_file_systems
    if resp and resp.file_systems
      resp.file_systems.each { |fs|
        found[fs.file_system_id] = fs
      }
    end
  end

  return found
end
isGlobal?() click to toggle source

Does this resource type exist as a global (cloud-wide) artifact, or is it localized to a region/zone? @return [Boolean]

# File modules/mu/providers/aws/storage_pool.rb, line 328
def self.isGlobal?
  false
end
modify_security_groups(cloud_id: nil, replace: false , security_groups: [], region: MU.curRegion) click to toggle source

Modify the security groups associated with an existing mount point @param cloud_id [String]: The cloud provider's identifier of the mount point. @param replace [TrueClass, FalseClass]: If the provided security groups will replace or be added to the existing ones @param security_groups [Array]: A list of security groups to associate with the mount point. @param region [String]: The cloud provider region

# File modules/mu/providers/aws/storage_pool.rb, line 239
def self.modify_security_groups(cloud_id: nil, replace: false , security_groups: [], region: MU.curRegion)
  unless replace
    extisting_sgs = MU::Cloud::AWS.efs(region: region).describe_mount_target_security_groups(
      mount_target_id: cloud_id
    ).security_groups

    security_groups.concat extisting_sgs
  end

  security_groups.uniq!
  MU::Cloud::AWS.efs(region: region).modify_mount_target_security_groups(
    mount_target_id: cloud_id,
    security_groups: security_groups
  )
end
new(**args) click to toggle source

Initialize this cloud resource object. Calling super will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us. @param args [Hash]: Hash of named arguments passed via Ruby's double-splat

Calls superclass method
# File modules/mu/providers/aws/storage_pool.rb, line 23
def initialize(**args)
  super
  @mu_name ||= @deploy.getResourceName(@config['name'])
end
quality() click to toggle source

Denote whether this resource implementation is experiment, ready for testing, or ready for production use.

# File modules/mu/providers/aws/storage_pool.rb, line 334
def self.quality
  MU::Cloud::RELEASE
end
schema(_config) click to toggle source

Cloud-specific configuration properties. @param _config [MU::Config]: The calling MU::Config object @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource

# File modules/mu/providers/aws/storage_pool.rb, line 438
def self.schema(_config)
  toplevel_required = []
  schema = {
    "ingress_rules" => {
      "type" => "array",
      "description" => "Firewall rules to apply to our mountpoints",
      "items" => {
        "type" => "object",
        "description" => "Firewall rules to apply to our mountpoints",
        "properties" => {
          "sgs" => {
            "type" => "array",
            "items" => {
              "description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
              "type" => "string"
            }
          },
          "lbs" => {
            "type" => "array",
            "items" => {
              "description" => "AWS Load Balancers which will have this rule applied to their traffic",
              "type" => "string"
            }
          }
        }
      }
    }
  }
  [toplevel_required, schema]
end
validateConfig(pool, configurator) click to toggle source

Cloud-specific pre-processing of {MU::Config::BasketofKittens::storage_pools}, bare and unvalidated. @param pool [Hash]: The resource to process and validate @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member @return [Boolean]: True if validation succeeded, False otherwise

# File modules/mu/providers/aws/storage_pool.rb, line 473
def self.validateConfig(pool, configurator)
  ok = true
  supported_regions = %w{us-west-2 us-east-1 us-east-2 eu-west-1}

  if !supported_regions.include?(pool['region'])
    MU.log "Region #{pool['region']} not supported. Only #{supported_regions.join(',  ')} are supported", MU::ERR
    ok = false
  end

  if pool['mount_points'] && !pool['mount_points'].empty?
    pool['mount_points'].each{ |mp|
      if mp['vpc'] and mp['vpc']['name']
        MU::Config.addDependency(pool, mp['vpc']['name'], "vpc")
      end
      if mp['ingress_rules']
        fwname = "storage-#{mp['name']}"
        acl = {
          "name" => fwname,
          "rules" => mp['ingress_rules'],
          "region" => pool['region'],
          "credentials" => pool['credentials'],
          "optional_tags" => pool['optional_tags']
        }
        acl["tags"] = pool['tags'] if pool['tags'] && !pool['tags'].empty?
        acl["vpc"] = mp['vpc'].dup if mp['vpc']
        ok = false if !configurator.insertKitten(acl, "firewall_rules")
        mp["add_firewall_rules"] = [] if mp["add_firewall_rules"].nil?
        mp["add_firewall_rules"] << {"name" => fwname}
      end
  
    }
  end

  ok
end

Public Instance Methods

addStandardTags(cloud_id: nil, region: MU.curRegion, credentials: nil) click to toggle source

Add our standard tag set to an Amazon EFS File System. @param cloud_id [String]: The cloud provider's identifier for this resource. @param region [String]: The cloud provider region

# File modules/mu/providers/aws/storage_pool.rb, line 154
def addStandardTags(cloud_id: nil, region: MU.curRegion, credentials: nil)
  if cloud_id
    tags = []
    MU::MommaCat.listStandardTags.each_pair { |name, value|
      tags << {key: name, value: value}
    }

    name_tag = false
    if @config['tags']
      @config['tags'].each { |tag|
        tags << {key: tag['key'], value: tag['value']}
        name_tag = true if tag['key'] == "Name"
      }
    end

    if @config['optional_tags']
      MU::MommaCat.listOptionalTags.each_pair { |name, value|
        tags << {key: name, value: value}
      }
    end

    tags << {key: "Name", value: @mu_name} unless name_tag

    MU::Cloud::AWS.efs(region: region, credentials: credentials).create_tags(
      file_system_id: cloud_id,
      tags: tags
    )
  else
    MU.log "cloud_id not provided, not tagging resources", MU::WARN
  end
end
arn() click to toggle source

Canonical Amazon Resource Number for this resource @return [String]

# File modules/mu/providers/aws/storage_pool.rb, line 102
def arn
  "arn:"+(MU::Cloud::AWS.isGovCloud?(@region) ? "aws-us-gov" : "aws")+":elasticfilesystem:"+@region+":"+MU::Cloud::AWS.credToAcct(@credentials)+":file-system/"+@cloud_id
end
create() click to toggle source

Called automatically by {MU::Deploy#createResources} @return [String]: The cloud provider's identifier for this storage pool.

# File modules/mu/providers/aws/storage_pool.rb, line 30
def create
  MU.log "Creating storage pool #{@mu_name}"
  resp = MU::Cloud::AWS.efs(region: @region, credentials: @credentials).create_file_system(
    creation_token: @mu_name,
    performance_mode: @config['storage_type']
  )

  attempts = 0
  loop do
    MU.log "Waiting for #{@mu_name}: #{resp.file_system_id} to become available" if attempts % 5 == 0
    storage_pool = MU::Cloud::AWS.efs(region: @region, credentials: @credentials).describe_file_systems(
      creation_token: @mu_name
    ).file_systems.first
    break if storage_pool.life_cycle_state == "available"
    raise MuError, "Failed to create storage pool #{@mu_name}" if %w{deleting deleted}.include? storage_pool.life_cycle_state
    sleep 10
    attempts += 1
    raise MuError, "timed out waiting for #{resp.mount_target_id }" if attempts >= 20
  end

  addStandardTags(cloud_id: resp.file_system_id, region: @region, credentials: @credentials)
  @cloud_id = resp.file_system_id

  if @config['mount_points'] && !@config['mount_points'].empty?
    mp_threads = []
    parent_thread_id = Thread.current.object_id
    @config['mount_points'].each { |target|
      sgs = []
      if target['add_firewall_rules']
        target['add_firewall_rules'].each { |mount_sg|
          sg = @deploy.findLitterMate(type: "firewall_rule", name: mount_sg['name'])
          sgs << sg.cloud_id if sg
        }
      end

      if target.has_key?("vpc") and target['vpc'].has_key?("vpc_name")
        vpc = @dependencies["vpc"][target['vpc']["vpc_name"]]
        if target['vpc']["subnet_name"]
          subnet_obj = vpc.getSubnet(name: target['vpc']["subnet_name"])
          if subnet_obj.nil?
            raise MuError, "Failed to locate subnet from #{target['vpc']["subnet_name"]} in StoragePool #{@config['name']}:#{target['name']}"
          end
          target['vpc']['subnet_id'] = subnet_obj.cloud_id
        end
      elsif target['vpc']["subnets"] and !target['vpc']["subnets"].empty?
        target['vpc']['subnet_id'] = target['vpc']["subnets"].first["subnet_id"]
      end

      mp_threads << Thread.new {
        MU.dupGlobals(parent_thread_id)
        mount_target = MU::Cloud::AWS::StoragePool.create_mount_target(
          cloud_id: @cloud_id,
          ip_address: target['ip_address'],
          subnet_id: target['vpc']['subnet_id'],
          security_groups: sgs,
          credentials: @credentials,
          region: @region
        )
        target['cloud_id'] = mount_target.mount_target_id
      }
    }

    mp_threads.each { |t|
      t.join
    }
  end

  return @cloud_id
end
groom() click to toggle source

Called automatically by {MU::Deploy#createResources}

# File modules/mu/providers/aws/storage_pool.rb, line 187
def groom
  #Nothing to do
end
notify() click to toggle source

Register a description of this storage pool with this deployment's metadata.

# File modules/mu/providers/aws/storage_pool.rb, line 256
def notify
  storage_pool = MU::Cloud::AWS.efs(region: @region, credentials: @credentials).describe_file_systems(
    creation_token: @mu_name
  ).file_systems.first

  targets = {}

  if @config['mount_points'] && !@config['mount_points'].empty?
    mount_targets = MU::Cloud::AWS.efs(region: @region, credentials: @credentials).describe_mount_targets(
      file_system_id: storage_pool.file_system_id
    ).mount_targets

    @config['mount_points'].each { |mp|
      subnet = nil
      dependencies
      mp_vpc = MU::Config::Ref.get(mp['vpc']).kitten


      subnet_obj = mp_vpc.subnets.select { |s|
        s.name == mp["vpc"]["subnet_name"] or s.cloud_id == mp["vpc"]["subnet_id"]
      }.first
      if !subnet_obj
        MU.log "Failed to find live subnet matching configured mount_point", MU::WARN, details: mp["vpc"]
        next
      end
      mount_target = nil
      mount_targets.each { |t|
        subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
        if subnet_cidr_obj.contains(NetAddr::IPv4.parse(t.ip_address))
          mount_target = t
          subnet = subnet_obj.cloud_desc
          break
        end
      }
      if !mount_target
        MU.log "Failed to find live mount_target corresponding to configured mount_point", MU::WARN, details: mp
        next
      end

      targets[mp["name"]] = {
        "owner_id" => mount_target.owner_id,
        "cloud_id" => mount_target.mount_target_id,
        "file_system_id" => mount_target.file_system_id,
        "mount_directory" => mp["directory"],
        "subnet_id" => mount_target.subnet_id,
        "vpc_id" => mp_vpc.cloud_id,
        "availability_zone" => subnet.availability_zone,
        "state" => mount_target.life_cycle_state,
        "ip_address" => mount_target.ip_address,
        "endpoint" => "#{subnet.availability_zone}.#{mount_target.file_system_id}.efs.#{@region}.amazonaws.com",
        "network_interface_id" => mount_target.network_interface_id
      }
    }
  end

  deploy_struct = {
    "owner_id" => storage_pool.owner_id,
    "creation_token" => storage_pool.creation_token,
    "identifier" => storage_pool.file_system_id,
    "creation_time" => storage_pool.creation_time,
    "number_of_mount_targets" => storage_pool.number_of_mount_targets,
    "size_in_bytes" => storage_pool.size_in_bytes.value,
    "type" => storage_pool.performance_mode,
    "mount_targets" => targets
  }

  return deploy_struct
end