class MU::Cloud::AWS

Support for Amazon Web Services as a provisioning layer.

Attributes

cloudformation_data[R]
region[R]

Public Class Methods

account_number() click to toggle source

Fetch the AWS account number where this Mu master resides. If it's not in AWS at all, or otherwise cannot be determined, return nil. here. XXX account for Google and non-cloud situations XXX this needs to be “myAccountNumber” or somesuch XXX and maybe do the IAM thing for arbitrary, non-resident accounts

# File modules/mu/providers/aws.rb, line 775
      def self.account_number
        require "aws-sdk-ec2"
        return nil if credConfig.nil?
        return @@my_acct_num if @@my_acct_num
        loadCredentials
# XXX take optional credential set argument

#       begin
#          user_list = MU::Cloud::AWS.iam(region: credConfig['region']).list_users.users
##        rescue ::Aws::IAM::Errors => e # XXX why does this NameError here?
#        rescue StandardError => e
#          MU.log "Got #{e.inspect} while trying to figure out our account number", MU::WARN, details: caller
#        end
#        if user_list.nil? or user_list.size == 0
          resp = MU::Cloud::AWS.getAWSMetaData("network/interfaces/macs/")
          return nil if !resp

          mac = resp.split(/\n/)[0]
          acct_num = MU::Cloud::AWS.getAWSMetaData("network/interfaces/macs/#{mac}owner-id")
          acct_num.chomp!
#        else
#          acct_num = MU::Cloud::AWS.iam(region: credConfig['region']).list_users.users.first.arn.split(/:/)[4]
#        end
        MU.setVar("acct_num", acct_num)
        @@my_acct_num ||= acct_num
        acct_num
      end
acm(region: MU.curRegion, credentials: nil) click to toggle source

Amazon Certificate Manager API

# File modules/mu/providers/aws.rb, line 1077
def self.acm(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@acm_api[credentials] ||= {}
  @@acm_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ACM", region: region, credentials: credentials)
  @@acm_api[credentials][region]
end
adminBucketName(credentials = nil) click to toggle source

Resolve the administrative S3 bucket for a given credential set, or return a default. @param credentials [String] @return [String]

# File modules/mu/providers/aws.rb, line 652
def self.adminBucketName(credentials = nil)
  require "aws-sdk-s3"
  cfg = credConfig(credentials)
  return nil if !cfg
  if !cfg['log_bucket_name']
    cfg['log_bucket_name'] = $MU_CFG['hostname']
    MU.log "No AWS log bucket defined for credentials #{credentials}, attempting to use default of #{cfg['log_bucket_name']}", MU::WARN
  end
  resp = MU::Cloud::AWS.s3(credentials: credentials).list_buckets
  found = false
  resp.buckets.each { |b|
    if b.name == cfg['log_bucket_name']
      found = true
      break
    end
  }
  if !found
    MU.log "Attempting to create log bucket #{cfg['log_bucket_name']} for credentials #{credentials}", MU::WARN
    begin
      MU::Cloud::AWS.s3(credentials: credentials).create_bucket(bucket: cfg['log_bucket_name'], acl: "private")
    rescue Aws::S3::Errors::BucketAlreadyExists
      raise MuError, "AWS credentials #{credentials} need a log bucket, and the name #{cfg['log_bucket_name']} is unavailable. Use mu-configure to edit credentials '#{credentials}' or 'hostname'"
    end
  end

  cfg['log_bucket_name']
end
adminBucketUrl(credentials = nil) click to toggle source

Resolve the administrative S3 bucket for a given credential set, or return a default. @param credentials [String] @return [String]

# File modules/mu/providers/aws.rb, line 684
def self.adminBucketUrl(credentials = nil)
  return nil if !credConfig(credentials)
  "s3://"+adminBucketName(credentials)+"/"
end
apig(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's API Gateway API

# File modules/mu/providers/aws.rb, line 1250
def self.apig(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@apig_api[credentials] ||= {}
  @@apig_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "APIGateway", region: region, credentials: credentials)
  @@apig_api[credentials][region]
end
attachedNVMeDisks() click to toggle source

If we're in AWS and NVME-aware, return a mapping of AWS-side device names to actual NVME devices. @return [Hash]

# File modules/mu/providers/aws.rb, line 476
def self.attachedNVMeDisks
  if !hosted? or !File.executable?("/bin/lsblk") or !File.executable?("/sbin/nvme")
    return {}
  end
  map = {}
  devices = MU::Master.listBlockDevices
  return {} if !devices
  devices.each { |d|
    if d =~ /^\/dev\/nvme/
      %x{/sbin/nvme id-ctrl -v #{d}}.each_line { |desc|
        if desc.match(/^0000: (?:[0-9a-f]{2} ){16}"(.+?)\./)
          virt_dev = Regexp.last_match[1]
          map[virt_dev] = d
          break
        end
      }
    end
  }
  map
end
autoscale(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's Autoscaling API

# File modules/mu/providers/aws.rb, line 1099
def self.autoscale(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@autoscale_api[credentials] ||= {}
  @@autoscale_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "AutoScaling", region: region, credentials: credentials)
  @@autoscale_api[credentials][region]
end
cleanDeploy(deploy_id, credentials: nil, noop: false) click to toggle source

Purge cloud-specific deploy meta-artifacts (SSH keys, resource groups, etc) @param deploy_id [MU::MommaCat]

# File modules/mu/providers/aws.rb, line 387
def self.cleanDeploy(deploy_id, credentials: nil, noop: false)

  if !noop
    MU.log "Deleting s3://#{adminBucketName(credentials)}/#{deploy_id}-secret"
    MU::Cloud::AWS.s3(credentials: credentials).delete_object(
      bucket: adminBucketName(credentials),
      key: "#{deploy_id}-secret"
    )
    listRegions(credentials: credentials).each { |r|
      resp = MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_key_pairs(
        filters: [{name: "key-name", values: ["deploy-#{MU.deploy_id}"]}]
      )
      resp.data.key_pairs.each { |keypair|
        MU.log "Deleting key pair #{keypair.key_name} from #{r}"
        MU::Cloud::AWS.ec2(region: r, credentials: credentials).delete_key_pair(key_name: keypair.key_name) if !noop
      }
    }

  end
  if hosted?
    MU::Cloud::AWS.openFirewallForClients
  end
end
cloudformation(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's CloudFormation API

# File modules/mu/providers/aws.rb, line 1137
def self.cloudformation(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@cloudformation_api[credentials] ||= {}
  @@cloudformation_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudFormation", region: region, credentials: credentials)
  @@cloudformation_api[credentials][region]
end
cloudfront(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's CloudFront API

# File modules/mu/providers/aws.rb, line 1202
def self.cloudfront(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@cloudfront_api[credentials] ||= {}
  @@cloudfront_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudFront", region: region, credentials: credentials)
  @@cloudfront_api[credentials][region]
end
cloudtrail(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's CloudTrail API

# File modules/mu/providers/aws.rb, line 1153
def self.cloudtrail(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@cloudtrail_api[credentials] ||= {}
  @@cloudtrail_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudTrail", region: region, credentials: credentials)
  @@cloudtrail_api[credentials][region]
end
cloudtrailBucketPolicy(credentials = nil) click to toggle source

Log bucket policy for enabling CloudTrail logging to our log bucket in S3.

# File modules/mu/providers/aws.rb, line 431
def self.cloudtrailBucketPolicy(credentials = nil)
  cfg = credConfig(credentials)
  policy_json = '{
          "Version": "2012-10-17",
          "Statement": [
                  {
                          "Sid": "AWSCloudTrailAclCheck20131101",
                          "Effect": "Allow",
        "Principal": {
          "AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::<%= MU.account_number %>:root",
          "Service": "cloudtrail.amazonaws.com"
        },
                          "Action": "s3:GetBucketAcl",
                          "Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'"
                  },
                  {
                          "Sid": "AWSCloudTrailWrite20131101",
                          "Effect": "Allow",
        "Principal": {
          "AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::'+credToAcct(credentials)+':root",
          "Service": "cloudtrail.amazonaws.com"
        },
                          "Action": "s3:PutObject",
                          "Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'/AWSLogs/'+credToAcct(credentials)+'/*",
                          "Condition": {
                                  "StringEquals": {
                                          "s3:x-amz-acl": "bucket-owner-full-control"
                                  }
                          }
                  }
          ]
  }'
  ERB.new(policy_json).result
end
cloudwatch(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's CloudWatch API

# File modules/mu/providers/aws.rb, line 1161
def self.cloudwatch(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@cloudwatch_api[credentials] ||= {}
  @@cloudwatch_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatch", region: region, credentials: credentials)
  @@cloudwatch_api[credentials][region]
end
cloudwatch_events(region = MU.cureRegion) click to toggle source

Amazon's Cloudwatch Events API

# File modules/mu/providers/aws.rb, line 1258
def self.cloudwatch_events(region = MU.cureRegion)
  region ||= myRegion
  @@cloudwatch_events_api[credentials] ||= {}
  @@cloudwatch_events_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatchEvents", region: region, credentials: credentials)
  @@cloudwatch_events_api
end
cloudwatchevents(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's CloudWatchEvents API

# File modules/mu/providers/aws.rb, line 1194
def self.cloudwatchevents(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@cloudwatchevents_api[credentials] ||= {}
  @@cloudwatchevents_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatchEvents", region: region, credentials: credentials)
  @@cloudwatchevents_api[credentials][region]
end
cloudwatchlogs(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's CloudWatchLogs API

# File modules/mu/providers/aws.rb, line 1186
def self.cloudwatchlogs(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@cloudwatchlogs_api[credentials] ||= {}
  @@cloudwatchlogs_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatchLogs", region: region, credentials: credentials)
  @@cloudwatchlogs_api[credentials][region]
end
cognito_ident(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's Cognito Identity API

# File modules/mu/providers/aws.rb, line 1322
def self.cognito_ident(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@cognito_ident_api[credentials] ||= {}
  @@cognito_ident_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CognitoIdentity", region: region, credentials: credentials)
  @@cognito_ident_api[credentials][region]
end
cognito_user(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's Cognito Identity Provider API

# File modules/mu/providers/aws.rb, line 1330
def self.cognito_user(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@cognito_user_api[credentials] ||= {}
  @@cognito_user_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CognitoIdentityProvider", region: region, credentials: credentials)
  @@cognito_user_api[credentials][region]
end
config_example() click to toggle source

A non-working example configuration

# File modules/mu/providers/aws.rb, line 573
      def self.config_example
        sample = hosted_config
        sample ||= {
          "region" => "us-east-1",
          "account_number" => "123456789012",
        }
#        sample["access_key"] = "AKIAIXKNI3JY6JVVJIHA"
#        sample["access_secret"] = "oWjHT+2N3veyswy7+UA5i+H14KpvrOIZlnRlxpkw"
        sample["credentials_file"] = "#{Etc.getpwuid(Process.uid).dir}/.aws/credentials"
        sample["log_bucket_name"] = "my-mu-s3-bucket"
        sample
      end
createEc2SSHKey(keyname, public_key, credentials: nil) click to toggle source

Generate an EC2 keypair unique to this deployment, given a regular OpenSSH-style public key and a name. @param keyname [String]: The name of the key to create. @param public_key [String]: The public key @return [Array<String>]: keypairname, ssh_private_key, ssh_public_key

# File modules/mu/providers/aws.rb, line 858
def self.createEc2SSHKey(keyname, public_key, credentials: nil)
  require "aws-sdk-ec2"
  # We replicate this key in all regions
  if !MU::Cloud::CloudFormation.emitCloudFormation
    MU::Cloud::AWS.listRegions.each { |region|
      MU.log "Replicating #{keyname} to EC2 in #{region}", MU::DEBUG, details: @ssh_public_key
      begin
        MU::Cloud::AWS.ec2(region: region, credentials: credentials).import_key_pair(
          key_name: keyname,
          public_key_material: public_key
        )
      rescue ::Aws::EC2::Errors::AuthFailure => e
        @@regions_semaphore.synchronize {
          @@regions.delete(region)
        }
        MU.log "#{region} threw #{e.message}, skipping", MU::ERR
      end
    }
  end
end
createStandardTags(resource = nil, region: MU.curRegion, credentials: nil, optional: true, nametag: nil, othertags: nil) click to toggle source

Tag an EC2 resource

@param resource [String]: The cloud provider identifier of the resource to tag @param region [String]: The cloud provider region @param credentials [String]: Credentials to authorize API requests @param optional [Boolean]: Whether to apply our optional generic tags @param nametag [String]: A Name tag to apply @param othertags [Array<Hash>]: Miscellaneous custom tags, in Basket of Kittens style @return [void]

# File modules/mu/providers/aws.rb, line 215
def self.createStandardTags(resource = nil, region: MU.curRegion, credentials: nil, optional: true, nametag: nil, othertags: nil)
  require "aws-sdk-ec2"
  tags = []
  MU::MommaCat.listStandardTags.each_pair { |name, value|
    tags << {key: name, value: value} if !value.nil?
  }
  if optional
    MU::MommaCat.listOptionalTags.each { |key, value|
      tags << {key: name, value: value} if !value.nil?
    }
  end
  if nametag
    tags << { key: "Name", value: nametag }
  end
  if othertags
    othertags.each { |tag|
      tags << { key: tag['key'], value: tag['value'] }
    }
  end

  if MU::Cloud::CloudFormation.emitCloudFormation
    return tags
  end

  attempts = 0
  begin
    MU::Cloud::AWS.ec2(region: region, credentials: credentials).create_tags(
      resources: [resource],
      tags: tags
    )
  rescue Aws::EC2::Errors::ServiceError => e
    MU.log "Got #{e.inspect} tagging #{resource} in #{region}, will retry", MU::WARN, details: caller.concat(tags) if attempts > 1
    if attempts < 5
      attempts = attempts + 1
      sleep 15
      retry
    else
      raise e
    end
  end
  MU.log "Created standard tags for resource #{resource}", MU::DEBUG, details: caller

end
createTag(resource = nil, tag_name="MU-ID", tag_value=MU.deploy_id, region: MU.curRegion, credentials: nil) click to toggle source

Tag a resource. Defaults to applying our MU deployment identifier, if no arguments other than the resource identifier are given.

@param resource [String]: The cloud provider identifier of the resource to tag @param tag_name [String]: The name of the tag to create @param tag_value [String]: The value of the tag @param region [String]: The cloud provider region @return [void]

# File modules/mu/providers/aws.rb, line 1389
def self.createTag(resource = nil,
    tag_name="MU-ID",
    tag_value=MU.deploy_id,
    region: MU.curRegion,
    credentials: nil)
  require "aws-sdk-ec2"
  attempts = 0

  return nil if resource.nil?
  resource = [resource] if resource.is_a?(String)

  if !MU::Cloud::CloudFormation.emitCloudFormation
    begin
      MU::Cloud::AWS.ec2(credentials: credentials, region: region).create_tags(
        resources: resource,
        tags: [
          {
            key: tag_name,
            value: tag_value
          }
        ]
      )
    rescue Aws::EC2::Errors::ServiceError => e
      MU.log "Got #{e.inspect} tagging #{resource} with #{tag_name}=#{tag_value}", MU::WARN if attempts > 1
      if attempts < 5
        attempts = attempts + 1
        sleep 15
        retry
      else
        raise e
      end
    end
    MU.log "Created tag #{tag_name} with value #{tag_value} for resource #{resource}", MU::DEBUG
  else
    return {
      "Key" =>  tag_name,
      "Value" => tag_value
    }
  end
end
credConfig(name = nil, name_only: false) click to toggle source

Return the $MU_CFG data associated with a particular profile/name/set of credentials. If no account name is specified, will return one flagged as default. Returns nil if AWS is not configured. Throws an exception if an account name is specified which does not exist. @param name [String]: The name of the key under 'aws' in mu.yaml to return @return [Hash,nil]

# File modules/mu/providers/aws.rb, line 695
      def self.credConfig(name = nil, name_only: false)
        # If there's nothing in mu.yaml (which is wrong), but we're running
        # on a machine hosted in AWS, *and* that machine has an IAM profile,
        # fake it with those credentials and hope for the best.
        if !$MU_CFG['aws'] or !$MU_CFG['aws'].is_a?(Hash) or $MU_CFG['aws'].size == 0
          if @@my_hosted_cfg
            return name_only ? "#default" : @@my_hosted_cfg
          end

          if hosted?
            begin
              iam_blob = getAWSMetaData("iam/info")
              if iam_blob
                iam_data = JSON.parse(iam_blob)
                if iam_data["InstanceProfileArn"] and !iam_data["InstanceProfileArn"].empty?
                  @@my_hosted_cfg = hosted_config
                  return name_only ? "#default" : @@my_hosted_cfg
                end
              end
            rescue JSON::ParserError => e
            end
          elsif ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY']
            env_config = {
              "region" => ENV['EC2_REGION'] || "us-east-1",
              "access_key" => ENV['AWS_ACCESS_KEY_ID'],
              "access_secret" => ENV['AWS_SECRET_ACCESS_KEY'],
              "log_bucket_name" => "mu-placeholder-bucket-name"
            }
            return name_only ? "#default" : env_config
          end

          return nil
        end

        if name.nil?
          $MU_CFG['aws'].each_pair { |set, cfg|
            if cfg['default']
              return name_only ? set : cfg
            end
          }
        else
          if $MU_CFG['aws'][name]
            return name_only ? name : $MU_CFG['aws'][name]
          elsif @@acct_to_profile_map[name.to_s]
            return name_only ? name : @@acct_to_profile_map[name.to_s]
          elsif name.is_a?(Integer) or name.match(/^\d+$/)
            # Try to map backwards from an account id, if that's what we go
            $MU_CFG['aws'].each_pair { |acctname, cfg|
              if cfg['account_number'] and name.to_s == cfg['account_number'].to_s
                return name_only ? acctname : $MU_CFG['aws'][acctname]
              end
            }

            # Check each credential sets' resident account, then
            $MU_CFG['aws'].each_pair { |acctname, cfg|
              begin
                MU::Cloud::AWS.iam(credentials: acctname).list_users.users
#             rescue ::Aws::IAM::Errors => e # XXX why does this NameError here?
              rescue StandardError => e
                MU.log e.inspect, MU::WARN, details: cfg
                next
              end
              acct_num = MU::Cloud::AWS.iam(credentials: acctname).list_users.users.first.arn.split(/:/)[4]
              cfg['account_number'] ||= acct_num.to_s
              if acct_num.to_s == name.to_s
                @@acct_to_profile_map[name.to_s] = cfg
                return name_only ? name.to_s : cfg
              end
            }
          end

          raise MuError, "AWS credential set #{name} was requested, but I see no such working credentials in mu.yaml"
        end
      end
credToAcct(name = nil) click to toggle source

Map the name of a credential set back to an AWS account number @param name [String]

# File modules/mu/providers/aws.rb, line 627
def self.credToAcct(name = nil)
  creds = credConfig(name)

  if creds['account_number'] and !creds['account_number'].empty?
    return creds['account_number']
  end

  acct_num = MU::Cloud::AWS.iam(credentials: name).list_users.users.first.arn.split(/:/)[4]
  acct_num.to_s
end
dynamo(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's DynamoDB API

# File modules/mu/providers/aws.rb, line 1282
def self.dynamo(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@dynamo_api[credentials] ||= {}
  @@dynamo_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "DynamoDB", region: region, credentials: credentials)
  @@dynamo_api[credentials][region]
end
dynamostream(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's DynamoStream API

# File modules/mu/providers/aws.rb, line 1290
def self.dynamostream(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@dynamostream_api[credentials] ||= {}
  @@dynamostream_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "DynamoDBStreams", region: region, credentials: credentials)
  @@dynamostream_api[credentials][region]
end
ec2(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's EC2 API

# File modules/mu/providers/aws.rb, line 1091
def self.ec2(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@ec2_api[credentials] ||= {}
  @@ec2_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "EC2", region: region, credentials: credentials)
  @@ec2_api[credentials][region]
end
ecs(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's ECS API

# File modules/mu/providers/aws.rb, line 1266
def self.ecs(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@ecs_api[credentials] ||= {}
  @@ecs_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ECS", region: region, credentials: credentials)
  @@ecs_api[credentials][region]
end
efs(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's EFS API

# File modules/mu/providers/aws.rb, line 1234
def self.efs(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@efs_api[credentials] ||= {}
  @@efs_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "EFS", region: region, credentials: credentials)
  @@efs_api[credentials][region]
end
eks(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's EKS API

# File modules/mu/providers/aws.rb, line 1274
def self.eks(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@eks_api[credentials] ||= {}
  @@eks_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "EKS", region: region, credentials: credentials)
  @@eks_api[credentials][region]
end
elasticache(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's ElastiCache API

# File modules/mu/providers/aws.rb, line 1210
def self.elasticache(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@elasticache_api[credentials] ||= {}
  @@elasticache_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ElastiCache", region: region, credentials: credentials)
  @@elasticache_api[credentials][region]
end
elasticsearch(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's Elasticsearch API

# File modules/mu/providers/aws.rb, line 1314
def self.elasticsearch(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@elasticsearch_api[credentials] ||= {}
  @@elasticsearch_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ElasticsearchService", region: region, credentials: credentials)
  @@elasticsearch_api[credentials][region]
end
elb(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's ElasticLoadBalancing API

# File modules/mu/providers/aws.rb, line 1107
def self.elb(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@elb_api[credentials] ||= {}
  @@elb_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ElasticLoadBalancing", region: region, credentials: credentials)
  @@elb_api[credentials][region]
end
elb2(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's ElasticLoadBalancingV2 (ALB) API

# File modules/mu/providers/aws.rb, line 1115
def self.elb2(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@elb2_api[credentials] ||= {}
  @@elb2_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ElasticLoadBalancingV2", region: region, credentials: credentials)
  @@elb2_api[credentials][region]
end
findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: nil, raise_on_missing: true) click to toggle source

AWS can stash API-available certificates in Amazon Certificate Manager or in IAM. Rather than make people crazy trying to get the syntax correct in our Baskets of Kittens, let's have a helper that tries to do the right thing, and only raise an exception if we need help to disambiguate. @param name [String]: The name of the cert. For IAM certs this can be any IAM name; for ACM, it's usually the domain name. If multiple matches are found, or no matches, an exception is raised. @param id [String]: The ARN of a known certificate. We just validate that it exists. This is ignored if a name parameter is supplied. @return [String]: The ARN of a matching certificate that is known to exist. If it is an ACM certificate, we also know that it is not expired.

# File modules/mu/providers/aws.rb, line 950
def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: nil, raise_on_missing: true)
  require "aws-sdk-iam"
  if (name.nil? or name.empty?) and (id.nil? or id.empty?)
    raise MuError, "Can't call findSSLCertificate without specifying either a name or an id"
  end
  if id and @@certificates[id]
    return [id, @@certificates[id]]
  end

  if !name.nil? and !name.empty?
    matches = []
    acmcerts = MU::Cloud::AWS.acm(region: region, credentials: credentials).list_certificates(
      certificate_statuses: ["ISSUED"]
    )
    acmcerts.certificate_summary_list.each { |cert|
      matches << cert.certificate_arn if cert.domain_name == name
    }
    begin
      iamcert = MU::Cloud::AWS.iam(credentials: credentials).get_server_certificate(
        server_certificate_name: name
      )
    rescue Aws::IAM::Errors::ValidationError, Aws::IAM::Errors::NoSuchEntity
      # valid names for ACM certs can break here, and that's ok to ignore
    end
    if !iamcert.nil?
      matches << iamcert.server_certificate.server_certificate_metadata.arn
    end
    if matches.size == 1
      id = matches.first
    elsif matches.size == 0
      if raise_on_missing
        raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}"
      else
        return nil
      end
    elsif matches.size > 1
      raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use."
    end
  end

  domains = []

  if id.match(/^arn:aws(?:-us-gov)?:acm/)
    resp = MU::Cloud::AWS.acm(region: region).describe_certificate(
      certificate_arn: id
    )

    if resp.nil? or resp.certificate.nil?
      raise MuError, "No such ACM certificate '#{id}'"
    end
    domains << resp.certificate.domain_name
    if resp.certificate.subject_alternative_names
      domains.concat(resp.certificate.subject_alternative_names)
    end
  elsif id.match(/^arn:aws(?:-us-gov)?:iam/)
    resp = MU::Cloud::AWS.iam.list_server_certificates
    if resp.nil?
      raise MuError, "No such IAM certificate '#{id}'"
    end
    resp.server_certificate_metadata_list.each { |cert|

      if cert.arn == id
        if cert.expiration < Time.now
          MU.log "IAM SSL certificate #{cert.server_certificate_name} (#{id}) is EXPIRED", MU::WARN
        end
        @@certificates[id] = [cert.server_certificate_name]
        return [id, [cert.server_certificate_name]]
      end
    }
    raise MuError, "No such IAM certificate '#{id}'"
  else
    raise MuError, "The format of '#{id}' doesn't look like an ARN for either Amazon Certificate Manager or IAM"
  end

  @@certificates[id] = domains.uniq
  [id, domains.uniq]
end
getAWSMetaData(param) click to toggle source

Fetch an Amazon instance metadata parameter (example: public-ipv4). @param param [String]: The parameter name to fetch @return [String, nil]

# File modules/mu/providers/aws.rb, line 1364
def self.getAWSMetaData(param)
  base_url = "http://169.254.169.254/latest/meta-data/"
  begin
    response = nil
    Timeout.timeout(1) do
      response = URI.open("#{base_url}/#{param}").read
    end

    response
  rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::ENETUNREACH, Net::HTTPServerException, Errno::EHOSTUNREACH => e
    # This is normal on machines checking to see if they're AWS-hosted
    logger = MU::Logger.new
    logger.log "Failed metadata request #{base_url}/#{param}: #{e.inspect}", MU::DEBUG
    return nil
  end
end
habitat(cloudobj, nolookup: false, deploy: nil) click to toggle source

Return what we think of as a cloud object's habitat. In AWS, this means the account_number in which it's resident. If this is not applicable, such as for a {Habitat} or {Folder}, returns nil. @param cloudobj [MU::Cloud::AWS]: The resource from which to extract the habitat id @return [String,nil]

# File modules/mu/providers/aws.rb, line 591
      def self.habitat(cloudobj, nolookup: false, deploy: nil)
        @@habmap ||= {}
# XXX whaddabout config['habitat'] HNNNGH

        if cloudobj.respond_to?(:account_number) and cloudobj.account_number and
           !cloudobj.account_number.empty?
          return cloudobj.account_number
        elsif cloudobj.config and cloudobj.config['account']
          if nolookup
            return cloudobj.config['account']
          end
          if @@habmap[cloudobj.config['account']]
            return @@habmap[cloudobj.config['account']]
          end
          deploy ||= cloudobj.deploy if cloudobj.respond_to?(:deploy)

          MU.log "Incomplete implementation: MU::Cloud::AWS.habitat", MU::DEBUG, details: deploy

#          accountobj = accountLookup(cloudobj.config['account'], deploy, raise_on_fail: false)

#          if accountobj
#            @@habmap[cloudobj.config['account']] = accountobj.cloud_id
#            return accountobj.cloud_id
#          end
        end

        nil
      end
hosted() click to toggle source

Alias for #{MU::Cloud::AWS.hosted?}

# File modules/mu/providers/aws.rb, line 469
def self.hosted
  MU::Cloud::AWS.hosted?
end
hosted?() click to toggle source

Determine whether we (the Mu master, presumably) are hosted in this cloud. @return [Boolean]

# File modules/mu/providers/aws.rb, line 523
def self.hosted?
  if $MU_CFG.has_key?("aws_is_hosted")
    @@is_in_aws = $MU_CFG["aws_is_hosted"]
    return $MU_CFG["aws_is_hosted"]
  end

  require 'open-uri'

  if !@@is_in_aws.nil?
    return @@is_in_aws
  end

  begin
    Timeout.timeout(4) do
      instance_id = URI.open("http://169.254.169.254/latest/meta-data/instance-id").read
      if !instance_id.nil? and instance_id.size > 0
        @@is_in_aws = true
        region = getAWSMetaData("placement/availability-zone").sub(/[a-z]$/i, "")
        begin
          validate_region(region)
        rescue MuError
          @@creds_loaded.delete("#default")
          @@is_in_aws = false
          false
        end
        return true
      end
    end
  rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::EHOSTUNREACH
  end

  @@is_in_aws = false
  false
end
hosted_config() click to toggle source

If we're running this cloud, return the $MU_CFG blob we'd use to describe this environment as our target one.

# File modules/mu/providers/aws.rb, line 560
def self.hosted_config
  return nil if !hosted?
  region = getAWSMetaData("placement/availability-zone").sub(/[a-z]$/i, "")
  mac = getAWSMetaData("network/interfaces/macs/").split(/\n/)[0]
  acct_num = getAWSMetaData("network/interfaces/macs/#{mac}owner-id")
  acct_num.chomp!
  {
    "region" => region,
    "account_number" => acct_num
  }
end
iam(credentials: nil) click to toggle source

Amazon's IAM API

# File modules/mu/providers/aws.rb, line 1085
def self.iam(credentials: nil)
  @@iam_api[credentials] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "IAM", credentials: credentials)
  @@iam_api[credentials]
end
initDeploy(deploy) click to toggle source

Do cloud-specific deploy instantiation tasks, such as copying SSH keys around, sticking secrets in buckets, creating resource groups, etc @param deploy [MU::MommaCat]

# File modules/mu/providers/aws.rb, line 381
def self.initDeploy(deploy)
end
isGovCloud?(region = myRegion) click to toggle source

Is the region we're dealing with a GovCloud region? @param region [String]: The region in question, defaults to the Mu Master's local region

# File modules/mu/providers/aws.rb, line 333
def self.isGovCloud?(region = myRegion)
  return false if !region
  region.match(/^us-gov-/)
end
kms(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's KMS API

# File modules/mu/providers/aws.rb, line 1338
def self.kms(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@kms_api[credentials] ||= {}
  @@kms_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "KMS", region: region, credentials: credentials)
  @@kms_api[credentials][region]
end
lambda(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's Lambda API

# File modules/mu/providers/aws.rb, line 1242
def self.lambda(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@lambda_api[credentials] ||= {}
  @@lambda_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "Lambda", region: region, credentials: credentials)
  @@lambda_api[credentials][region]
end
listAZs(region: MU.curRegion, credentials: nil) click to toggle source

List the Availability Zones associated with a given Amazon Web Services region. If no region is given, search the one in which this MU master server resides. @param region [String]: The region to search. @return [Array<String>]: The Availability Zones in this region.

# File modules/mu/providers/aws.rb, line 360
def self.listAZs(region: MU.curRegion, credentials: nil)
  cfg = credConfig(credentials)
  return [] if !cfg
  if !region.nil? and @@azs[region]
    return @@azs[region]
  end
  if region
    azs = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_availability_zones(
      filters: [name: "region-name", values: [region]]
    )
  end
  @@azs[region] ||= []
  azs.data.availability_zones.each { |az|
    @@azs[region] << az.zone_name if az.state == "available"
  }
  return @@azs[region]
end
listCredentials() click to toggle source

Return the name strings of all known sets of credentials for this cloud @return [Array<String>]

# File modules/mu/providers/aws.rb, line 640
def self.listCredentials
  if !$MU_CFG['aws']
    return hosted? ? ["#default"] : nil
  end

  $MU_CFG['aws'].keys
end
listHabitats(credentials = nil, use_cache: true) click to toggle source

List all AWS projects available to our credentials

# File modules/mu/providers/aws.rb, line 41
def self.listHabitats(credentials = nil, use_cache: true)
  cfg = credConfig(credentials)
  return [] if !cfg or !cfg['account_number']
  [cfg['account_number']]
end
listInstanceTypes(region = myRegion) click to toggle source

Query the AWS API for the list of valid EC2 instance types and some of their attributes. We can use this in config validation and to help “translate” machine types across cloud providers. @param region [String]: Supported machine types can vary from region to region, so we look for the set we're interested in specifically @return [Hash]

# File modules/mu/providers/aws.rb, line 885
def self.listInstanceTypes(region = myRegion)
  return @@instance_types if @@instance_types and @@instance_types[region]
  return {} if credConfig.nil?
  if region.nil?
    region = myRegion(debug: true)
  end
  return {} if region.nil?

  human_region = @@regionLookup[region]
  if human_region.nil?
    MU.log "Failed to map a Pricing API region name from #{region}", MU::ERR
    return {}
  end

  @@instance_types ||= {}
  @@instance_types[region] ||= {}

  # Pricing API isn't widely available, so ask a region we know supports
  # it
  resp = MU::Cloud::AWS.pricing(region: "us-east-1").get_products(
    service_code: "AmazonEC2",
    filters: [
      {
        field: "productFamily",
        value: "Compute Instance",
        type: "TERM_MATCH"
      },
      {
        field: "tenancy",
        value: "Shared",
        type: "TERM_MATCH"
      },
      {
        field: "location",
        value: human_region,
        type: "TERM_MATCH"
      }
    ]
  )
  resp.price_list.each { |pricing|
    data = JSON.parse(pricing)
    type = data["product"]["attributes"]["instanceType"]
    next if @@instance_types[region].has_key?(type)
    @@instance_types[region][type] = {}
    ["ecu", "vcpu", "memory", "storage"].each { |a|
      @@instance_types[region][type][a] = data["product"]["attributes"][a]
    }
    @@instance_types[region][type]["memory"].sub!(/ GiB/, "")
    @@instance_types[region][type]["memory"] = @@instance_types[region][type]["memory"].to_f
    @@instance_types[region][type]["vcpu"] = @@instance_types[region][type]["vcpu"].to_f
  }

  @@instance_types
end
listRegions(us_only = false, credentials: nil) click to toggle source

List the Amazon Web Services region names available to this account. The region that is local to this Mu server will be listed first. @param us_only [Boolean]: Restrict results to United States only @return [Array<String>]

# File modules/mu/providers/aws.rb, line 809
      def self.listRegions(us_only = false, credentials: nil)

        if @@regions.size == 0
          return [] if credConfig.nil?
          result = MU::Cloud::AWS.ec2(region: myRegion, credentials: credentials).describe_regions.regions
          @@regions_semaphore.synchronize {
            begin
              result.each { |r|
                @@regions[r.region_name] = Proc.new {
                  listAZs(region: r.region_name, credentials: credentials)
                }
              }
            rescue ::Aws::EC2::Errors::AuthFailure => e
              MU.log "Region #{r.region_name} throws #{e.message}, ignoring it", MU::ERR
            end
          }
        end


        regions = if us_only
          @@regions.keys.delete_if { |r| !r.match(/^us\-/) }.uniq
        else
          @@regions.keys.uniq
        end

# XXX GovCloud doesn't show up if you query a commercial endpoint... that's
# *probably* ok for most purposes? We can't call listAZs on it from out here
# apparently, so getting around it is nontrivial
#        if !@@regions.has_key?("us-gov-west-1")
#          @@regions["us-gov-west-1"] = Proc.new { listAZs("us-gov-west-1") }
#        end

        regions.sort! { |a, b|
          val = a <=> b
          if a == myRegion
            val = -1
          elsif b == myRegion
            val = 1
          end
          val
        }
        regions
      end
loadCredentials(name = nil) click to toggle source

Load some credentials for using the AWS API @param name [String]: The name of the mu.yaml AWS credential set to use. If not specified, will use the default credentials, and set the global Aws.config credentials to those. @return [Aws::Credentials]

# File modules/mu/providers/aws.rb, line 67
      def self.loadCredentials(name = nil)
        gem 'aws-sdk-core'
        @@creds_loaded ||= {}

        if name.nil?
          return @@creds_loaded["#default"] if @@creds_loaded["#default"]
        else
          return @@creds_loaded[name] if @@creds_loaded[name]
        end

        cred_cfg = credConfig(name)
        if cred_cfg.nil?
          return nil
        end

        cred_obj = nil
        if cred_cfg['access_key'] and cred_cfg['access_secret'] and
          # access key and secret just sitting in mu.yaml
           !cred_cfg['access_key'].empty? and
           !cred_cfg['access_secret'].empty?
          cred_obj = Aws::Credentials.new(
            cred_cfg['access_key'], cred_cfg['access_secret']
          )
          if name.nil?
#            Aws.config = {
#              access_key_id: cred_cfg['access_key'],
#              secret_access_key: cred_cfg['access_secret'],
#              region: cred_cfg['region']
#            }
          end
        elsif cred_cfg['credentials_file'] and
              !cred_cfg['credentials_file'].empty?

          # pull access key and secret from an awscli-style credentials file
          begin
            File.read(cred_cfg["credentials_file"]) # make sure it's there
            credfile = IniFile.load(cred_cfg["credentials_file"])

            if !credfile.sections or credfile.sections.size == 0
              raise ::IniFile::Error, "No AWS profiles found in #{cred_cfg["credentials_file"]}"
            end
            data = credfile.has_section?("default") ? credfile["default"] : credfile[credfile.sections.first]
            if data["aws_access_key_id"] and data["aws_secret_access_key"]
              cred_obj = Aws::Credentials.new(
                data['aws_access_key_id'], data['aws_secret_access_key']
              )
              if name.nil?
#                Aws.config = {
#                  access_key_id: data['aws_access_key_id'],
#                  secret_access_key: data['aws_secret_access_key'],
#                  region: cred_cfg['region']
#                }
              end
            else
              MU.log "AWS credentials in #{cred_cfg["credentials_file"]} specified, but is missing aws_access_key_id or aws_secret_access_key elements", MU::WARN
            end
          rescue IniFile::Error, Errno::ENOENT, Errno::EACCES => e
            MU.log "AWS credentials file #{cred_cfg["credentials_file"]} is missing or invalid", MU::WARN, details: e.message
          end
        elsif cred_cfg['credentials'] and
              !cred_cfg['credentials'].empty?
          # pull access key and secret from a vault
          begin
            vault, item = cred_cfg["credentials"].split(/:/)
            data = if !vault or !item
              raise MuError.new "AWS #{name} credentials field value '#{cred_cfg["credentials"]}' malformed, should be vaultname:itemname", details: cred_cfg
            else
              MU::Groomer::Chef.getSecret(vault: vault, item: item).to_h
            end
            if data and data["access_key"] and data["access_secret"]
              cred_obj = Aws::Credentials.new(
                data['access_key'], data['access_secret']
              )
              if name.nil?
#                Aws.config = {
#                  access_key_id: data['access_key'],
#                  secret_access_key: data['access_secret'],
#                  region: cred_cfg['region']
#                }
              end
            else
              raise MuError.new "AWS #{name} credentials vault:item #{cred_cfg["credentials"]} specified, but is missing access_key or access_secret elements", details: cred_cfg
            end
          rescue MU::Groomer::MuNoSuchSecret
            raise MuError.new "AWS #{name} credentials vault:item #{cred_cfg["credentials"]} specified, but does not exist", details: cred_cfg
          end
        end

        if !cred_obj and hosted?
          # assume we've got an IAM profile and hope for the best
          ENV.delete('AWS_ACCESS_KEY_ID')
          ENV.delete('AWS_SECRET_ACCESS_KEY')
          retries = 0
          begin
            cred_obj = Aws::InstanceProfileCredentials.new
            if cred_obj.nil?
              retries += 1
              MU.log "Failed to fetch AWS instance profile credentials, attempt #{retries.to_s}/10", MU::WARN
              sleep 3
            end
          end while cred_obj.nil? and retries < 10
#          if name.nil?
#            Aws.config = {region: ENV['EC2_REGION']}
#          end
        end
if cred_obj.nil?
MU.log "cred_obj is nil and hosted? says #{hosted?.to_s}", MU::WARN, details: name
end

        if name.nil?
          @@creds_loaded["#default"] = cred_obj
        else
          @@creds_loaded[name] = cred_obj
        end

        cred_obj
      end
myRegion(credentials = nil, debug: false) click to toggle source

If we've configured AWS as a provider, or are simply hosted in AWS, decide what our default region is.

# File modules/mu/providers/aws.rb, line 276
def self.myRegion(credentials = nil, debug: false)
  loglevel = debug ? MU::NOTICE : MU::DEBUG
  if @@myRegion_var
    MU.log "AWS.myRegion: returning #{@@myRegion_var} from cache", loglevel
    return @@myRegion_var
  end

  MU.log "AWS.myRegion: credConfig", loglevel, details: credConfig
  MU.log "AWS.myRegion: hosted?", loglevel, details: hosted?.to_s
  MU.log "AWS.myRegion: ENV['EC2_REGION']", loglevel, details: ENV['EC2_REGION']
  MU.log "AWS.myRegion: $MU_CFG['aws']", loglevel, details: $MU_CFG['aws']
  if credConfig.nil? and !hosted? and !ENV['EC2_REGION']
    MU.log "AWS.myRegion: nothing of use set, returning", loglevel
    return nil
  end

  if $MU_CFG and $MU_CFG['aws']
    $MU_CFG['aws'].each_pair { |credset, cfg|
      MU.log "AWS.myRegion: #{credset} != #{credentials} ?", loglevel, details: cfg
      next if credentials and credset != credentials
      MU.log "AWS.myRegion: validating credset #{credset}", loglevel, details: cfg
      next if !cfg['region']
      MU.log "AWS.myRegion: validation response", loglevel, details: validate_region(cfg['region'], credentials: credset)
      if (cfg['default'] or !@@myRegion_var or $MU_CFG['aws'].size == 1) and validate_region(cfg['region'], credentials: credset)
        MU.log "AWS.myRegion: liking this set", loglevel, details: cfg
        @@myRegion_var = cfg['region']
        break if cfg['default'] or credentials
      end
    }
  elsif ENV.has_key?("EC2_REGION") and !ENV['EC2_REGION'].empty? and
        validate_region(ENV['EC2_REGION']) and
        (
         (ENV.has_key?("AWS_SECRET_ACCESS_KEY") and ENV.has_key?("AWS_SECRET_ACCESS_KEY") ) or
         (Aws.config['access_key'] and Aws.config['access_secret'])
        )
    # Make sure this string is valid by way of the API
    MU.log "AWS.myRegion: using ENV", loglevel, details: ENV
    @@myRegion_var = ENV['EC2_REGION']
  end

  if hosted? and !@@myRegion_var
    # hacky, but useful in a pinch (and if we're hosted in AWS)
    az_str = MU::Cloud::AWS.getAWSMetaData("placement/availability-zone")
    MU.log "AWS.myRegion: using hosted", loglevel, details: az_str
    @@myRegion_var = az_str.sub(/[a-z]$/i, "") if az_str
  end

  if credConfig and credConfig["region"]
    @@myRegion_var ||= credConfig["region"]
  end

  @@myRegion_var
end
myVPCObj() click to toggle source

If we reside in this cloud, return the VPC in which we, the Mu Master, reside. @return [MU::Cloud::VPC]

# File modules/mu/providers/aws.rb, line 263
def self.myVPCObj
  return @@myVPCObj if @@myVPCObj
  return nil if !hosted?
  instance = MU.myCloudDescriptor
  return nil if !instance or !instance.vpc_id
  vpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: instance.vpc_id, dummy_ok: true, no_deploy_search: true)
  return nil if vpc.nil? or vpc.size == 0
  @@myVPCObj = vpc.first
  @@myVPCObj
end
nameMatchesCertificate(name, cert_id) click to toggle source

Given a domain name and an ACM or IAM certificate identifier, sort out whether the domain name is “covered” by the certificate @param name [String] @param cert_id [String] @return [Boolean]

# File modules/mu/providers/aws.rb, line 1033
def self.nameMatchesCertificate(name, cert_id)
  _id, domains = findSSLCertificate(id: cert_id)
  return false if !domains
  domains.each { |dom|
    if dom == name or
       (dom =~ /^\*/ and name =~ /.*#{Regexp.quote(dom[1..-1])}/)
      return true
    end
  }
  false
end
openFirewallForClients() click to toggle source

Punch AWS security group holes for client nodes to talk back to us, the Mu Master, if we're in AWS. @return [void]

# File modules/mu/providers/aws.rb, line 1434
      def self.openFirewallForClients
        require "aws-sdk-ec2"
        MU::Cloud.resourceClass("AWS", :FirewallRule)
        begin
          if File.exist?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
            ::Chef::Config.from_file(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
          end
          ::Chef::Config[:environment] = MU.environment
        rescue LoadError
          # XXX why is Chef here
        end

        # This is the set of (TCP) ports we're opening to clients. We assume that
        # we can and and remove these without impacting anything a human has
        # created.

        my_ports = [10514]

        my_instance_id = MU::Cloud::AWS.getAWSMetaData("instance-id")
        my_client_sg_name = "Mu Client Rules for #{MU.mu_public_ip}"
        my_sgs = Array.new

        MU.setVar("curRegion", myRegion) if !myRegion.nil?

        MU.myCloudDescriptor.security_groups.each { |sg|
          my_sgs << sg.group_id
        }
        resp = MU::Cloud::AWS.ec2.describe_security_groups(
          filters: [
            {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]},
            {name: "tag:Name", values: [my_client_sg_name]}
          ]
        )

        if resp.nil? or resp.security_groups.nil? or resp.security_groups.size == 0
          if MU.myCloudDescriptor.vpc_id.nil?
            sg_id = my_sgs.first
            resp = MU::Cloud::AWS.ec2.describe_security_groups(group_ids: [sg_id])
            group = resp.security_groups.first
            MU.log "We don't have a security group named '#{my_client_sg_name}' available, and we are in EC2 Classic and so cannot create a new group. Defaulting to #{group.group_name}.", MU::NOTICE
          else
            group = MU::Cloud::AWS.ec2.create_security_group(
              group_name: my_client_sg_name,
              description: my_client_sg_name,
              vpc_id: MU.myCloudDescriptor.vpc_id
            )
            sg_id = group.group_id
            my_sgs << sg_id
            MU::Cloud::AWS.createTag sg_id, "Name", my_client_sg_name
            MU::Cloud::AWS.createTag sg_id, "MU-MASTER-IP", MU.mu_public_ip
            MU::Cloud::AWS.ec2.modify_instance_attribute(
                instance_id: my_instance_id,
                groups: my_sgs
            )
          end
        elsif resp.security_groups.size == 1
          sg_id = resp.security_groups.first.group_id
          resp = MU::Cloud::AWS.ec2.describe_security_groups(group_ids: [sg_id])
          group = resp.security_groups.first
        else
          MU.log "Found more than one security group named #{my_client_sg_name}, aborting", MU::ERR
          exit 1
        end

        if !my_sgs.include?(sg_id)
          my_sgs << sg_id
          MU.log "Associating #{my_client_sg_name} with #{MU.myInstanceId}", MU::NOTICE
          MU::Cloud::AWS.ec2.modify_instance_attribute(
            instance_id: MU.myInstanceId,
            groups: my_sgs
          )
        end

        begin
          MU.log "Using AWS Security Group '#{group.group_name}' (#{sg_id})"
        rescue NoMethodError
          MU.log "Using AWS Security Group #{sg_id}"
        end

        allow_ips = ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]
        MU::MommaCat.listAllNodes.values.each { |data|
          next if data.nil? or !data.is_a?(Hash)
          ["public_ip_address"].each { |key|
            if data.has_key?(key) and !data[key].nil? and !data[key].empty?
              allow_ips << data[key] + "/32"
            end
          }
        }
        allow_ips.uniq!

        @syslog_port_semaphore.synchronize {
          my_ports.each { |port|
            begin
              group.ip_permissions.each { |rule|
                if rule.ip_protocol == "tcp" and
                    rule.from_port == port and rule.to_port == port
                  MU.log "Revoking old rules for port #{port.to_s} from #{sg_id}", MU::NOTICE
                  begin
                    MU::Cloud::AWS.ec2(region: myRegion).revoke_security_group_ingress(
                        group_id: sg_id,
                        ip_permissions: [
                            {
                                ip_protocol: "tcp",
                                from_port: port,
                                to_port: port,
                                ip_ranges: MU.structToHash(rule.ip_ranges)
                            }
                        ]
                    )
                  rescue Aws::EC2::Errors::InvalidPermissionNotFound
                    MU.log "Permission disappeared from #{sg_id} (port #{port.to_s}) before I could remove it", MU::WARN, details: MU.structToHash(rule.ip_ranges)
                  end
                end
              }
            rescue NoMethodError
# XXX this is ok
            end
            MU.log "Adding current IP list to allow rule for port #{port.to_s} in #{sg_id}", details: allow_ips

            allow_ips_cidr = []
            allow_ips.each { |cidr|
              allow_ips_cidr << {"cidr_ip" => cidr}
            }

            begin
              MU::Cloud::AWS.ec2(region: myRegion).authorize_security_group_ingress(
                  group_id: sg_id,
                  ip_permissions: [
                      {
                          ip_protocol: "tcp",
                          from_port: 10514,
                          to_port: 10514,
                          ip_ranges: allow_ips_cidr
                      }
                  ]
              )
            rescue Aws::EC2::Errors::InvalidPermissionDuplicate => e
              MU.log "Got #{e.inspect} in MU::Cloud::AWS.openFirewallForClients", MU::WARN, details: allow_ips_cidr
            end
          }
        }
      end
orgs(credentials: nil) click to toggle source

Amazon's Organizations API

# File modules/mu/providers/aws.rb, line 1354
      def self.orgs(credentials: nil)
        @@organizations_api ||= {}
# XXX org api doesn't seem to work in many regions
        @@organizations_api[credentials] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "Organizations", credentials: credentials, region: "us-east-1")
        @@organizations_api[credentials]
      end
pricing(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's Pricing API

# File modules/mu/providers/aws.rb, line 1298
def self.pricing(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@pricing_api[credentials] ||= {}
  @@pricing_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "Pricing", region: region, credentials: credentials)
  @@pricing_api[credentials][region]
end
rds(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's RDS API

# File modules/mu/providers/aws.rb, line 1129
def self.rds(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@rds_api[credentials] ||= {}
  @@rds_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "RDS", region: region, credentials: credentials)
  @@rds_api[credentials][region]
end
realDevicePath(dev) click to toggle source

Map our own idea of what a block device is called back to whatever AWS and the operating system decided on amongst themselves. This currently exists to map generic “xvd” style names back to real NVMe devices. @param dev [String]

# File modules/mu/providers/aws.rb, line 501
def self.realDevicePath(dev)
  return dev if !hosted?
  value = nil
  should_retry = Proc.new {
    !value and MU::Master.nvme?
  }
  MU.retrier(loop_if: should_retry, wait: 5, max: 6) {
    map = attachedNVMeDisks
    value = if map[dev]
      map[dev]
    elsif map[dev.gsub(/.*?\//, '')]
      map[dev.gsub(/.*?\//, '')]
    else
      dev # be nice to actually handle this too
    end
  }
  value
end
removeTag(key, value, resources = [], region: myRegion) click to toggle source

@param resources [Array<String>]: The cloud provider identifier of the resource to untag @param key [String]: The name of the tag to remove @param value [String]: The value of the tag to remove @param region [String]: The cloud provider region

# File modules/mu/providers/aws.rb, line 342
def self.removeTag(key, value, resources = [], region: myRegion)
  MU::Cloud::AWS.ec2(region: region).delete_tags(
    resources: resources,
    tags: [
      {
        key: key,
        value: value
      }
    ]
  )
end
required_instance_methods() click to toggle source

Any cloud-specific instance methods we require our resource implementations to have, above and beyond the ones specified by {MU::Cloud} @return [Array<Symbol>]

# File modules/mu/providers/aws.rb, line 189
def self.required_instance_methods
  [:arn]
end
resolveSSLCertificate(certblock, region: nil, credentials: nil) click to toggle source

Given a {MU::Config::Ref} block for an IAM or ACM SSL certificate, look up and validate the specified certificate. This is intended to be invoked from resource implementations' validateConfig methods. @param certblock [Hash,MU::Config::Ref]: @param region [String]: Default region to use when looking up the certificate, if its configuration block does not specify any @param credentials [String]: Default credentials to use when looking up the certificate, if its configuration block does not specify any @return [Boolean]

# File modules/mu/providers/aws.rb, line 1052
def self.resolveSSLCertificate(certblock, region: nil, credentials: nil)
  return false if !certblock
  ok = true

  certblock['region'] ||= region if !certblock['id']
  certblock['credentials'] ||= credentials
  cert_arn, cert_domains = MU::Cloud::AWS.findSSLCertificate(
    name: certblock["name"],
    id: certblock["id"],
    region: certblock['region'],
    credentials: certblock['credentials']
  )

  if cert_arn
    certblock['id'] ||= cert_arn
  end

  ['region', 'credentials'].each { |field|
    certblock.delete(field) if certblock[field].nil?
  }

  [cert_arn, cert_domains]
end
resourceInitHook(cloudobj, _deploy) click to toggle source

A hook that is always called just before any of the instance method of our resource implementations gets invoked, so that we can ensure that repetitive setup tasks (like resolving :resource_group for Azure resources) have always been done. @param cloudobj [MU::Cloud] @param _deploy [MU::MommaCat]

# File modules/mu/providers/aws.rb, line 53
def self.resourceInitHook(cloudobj, _deploy)
  class << self
    attr_reader :cloudformation_data
    attr_reader :region
  end
  return if !cloudobj
  cloudobj.instance_variable_set(:@cloudformation_data, {})

  cloudobj.instance_variable_set(:@region, cloudobj.config['region'])
end
route53(credentials: nil) click to toggle source

Amazon's Route53 API

# File modules/mu/providers/aws.rb, line 1123
def self.route53(credentials: nil)
  @@route53_api[credentials] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "Route53", credentials: credentials)
  @@route53_api[credentials]
end
s3(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's S3 API

# File modules/mu/providers/aws.rb, line 1145
def self.s3(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@s3_api[credentials] ||= {}
  @@s3_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "S3", region: region, credentials: credentials)
  @@s3_api[credentials][region]
end
sns(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's SNS API

# File modules/mu/providers/aws.rb, line 1218
def self.sns(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@sns_api[credentials] ||= {}
  @@sns_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "SNS", region: region, credentials: credentials)
  @@sns_api[credentials][region]
end
sqs(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's SQS API

# File modules/mu/providers/aws.rb, line 1226
def self.sqs(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@sqs_api[credentials] ||= {}
  @@sqs_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "SQS", region: region, credentials: credentials)
  @@sqs_api[credentials][region]
end
ssm(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's Simple Systems Manager API

# File modules/mu/providers/aws.rb, line 1306
def self.ssm(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@ssm_api[credentials] ||= {}
  @@ssm_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "SSM", region: region, credentials: credentials)
  @@ssm_api[credentials][region]
end
validate_region(r, credentials: nil) click to toggle source

Given an AWS region, check the API to make sure it's a valid one @param r [String] @return [String]

# File modules/mu/providers/aws.rb, line 196
def self.validate_region(r, credentials: nil)
  require "aws-sdk-ec2"
  begin
    MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_availability_zones.availability_zones.first.region_name
  rescue ::Aws::EC2::Errors::UnauthorizedOperation => e
    MU.log "Got '#{e.message}' trying to validate region #{r} (hosted: #{hosted?.to_s})", MU::ERR, details: loadCredentials(credentials)
    raise MuError, "Got '#{e.message}' trying to validate region #{r} with credentials #{credentials ? credentials : "<default>"} (hosted: #{hosted?.to_s})"
  end
end
virtual?() click to toggle source

Is this a “real” cloud provider, or a stub like CloudFormation?

# File modules/mu/providers/aws.rb, line 36
def self.virtual?
  false
end
waf(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's Web Application Firewall API (Regional, for ALBs et al)

# File modules/mu/providers/aws.rb, line 1178
def self.waf(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@waf[credentials] ||= {}
  @@waf[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "WAFRegional", region: region, credentials: credentials)
  @@waf[credentials][region]
end
wafglobal(region: MU.curRegion, credentials: nil) click to toggle source

Amazon's Web Application Firewall API (Global, for CloudFront et al)

# File modules/mu/providers/aws.rb, line 1169
def self.wafglobal(region: MU.curRegion, credentials: nil)
  region ||= myRegion
  @@wafglobal_api[credentials] ||= {}
  @@wafglobal[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "WAF", region: region, credentials: credentials)
  @@wafglobal[credentials][region]
end
writeDeploySecret(deploy, value, name = nil, credentials: nil) click to toggle source

Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it @param deploy_id [String]: The deploy for which we're writing the secret @param value [String]: The contents of the secret

# File modules/mu/providers/aws.rb, line 414
def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
  require "aws-sdk-s3"
  name ||= deploy.deploy_id+"-secret"
  begin
    MU.log "Writing #{name} to S3 bucket #{adminBucketName(credentials)}"
    MU::Cloud::AWS.s3(region: myRegion, credentials: credentials).put_object(
      acl: "private",
      bucket: adminBucketName(credentials),
      key: name,
      body: value
    )
  rescue Aws::S3::Errors => e
    raise MU::MommaCat::DeployInitializeError, "Got #{e.inspect} trying to write #{name} to #{adminBucketName(credentials)}"
  end
end