class MU::Cloud::AWS::SearchDomain

A search_domain as configured in {MU::Config::BasketofKittens::search_domains}

Public Class Methods

cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {}) click to toggle source

Remove all search_domains associated with the currently loaded deployment. @param noop [Boolean]: If true, will only print what would be done @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server @param region [String]: The cloud provider region @return [void]

# File modules/mu/providers/aws/search_domain.rb, line 123
        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
          MU.log "AWS::SearchDomain.cleanup: need to support flags['known']", MU::DEBUG, details: flags

          list = MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).list_domain_names
          if list and list.domain_names and list.domain_names.size > 0
            names = list.domain_names.map { |d| d.domain_name }
            begin
              # why is this API so obnoxious?
              sample = names.slice!(0, (names.length >= 5 ? 5 : names.length))
              descs = MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).describe_elasticsearch_domains(domain_names: sample)

              descs.domain_status_list.each { |domain|
                tags = MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).list_tags(arn: domain.arn)
                deploy_match = false
                master_match = false
                tags.tag_list.each { |tag|
                  if tag.key == "MU-ID" and tag.value == deploy_id
                    deploy_match = true
                  elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
                    master_match = true
                  end
                }
                if deploy_match and (master_match or ignoremaster)
                  MU.log "Deleting ElasticSearch Domain #{domain.domain_name}"
                  if !noop
                    MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).delete_elasticsearch_domain(domain_name: domain.domain_name)
                  end
                end
              }
            end while names.size > 0
          end

          unless noop
            marker = nil
            begin
              resp = MU::Cloud::AWS.iam(credentials: credentials).list_roles(marker: marker)
              resp.roles.each{ |role|
                # XXX Maybe we should have a more generic way to delete IAM profiles and policies. The call itself should be moved from MU::Cloud.resourceClass("AWS", "Server").
#                MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(role.role_name) if role.role_name.match(/^#{Regexp.quote(deploy_id)}/)
              }
              marker = resp.marker
            end while resp.is_truncated
          end
        end
find(**args) click to toggle source

Locate an existing search_domain. @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching search_domain.

# File modules/mu/providers/aws/search_domain.rb, line 170
def self.find(**args)
  found = {}

  # Annoyingly, we might expect one of several possible artifacts,
  # since AWS couldn't decide what the real identifier of these
  # things should be
  list = MU::Cloud::AWS.elasticsearch(region: args[:region], credentials: args[:credentials]).list_domain_names
  if list and list.domain_names and list.domain_names.size > 0
    descs = MU::Cloud::AWS.elasticsearch(region: args[:region], credentials: args[:credentials]).describe_elasticsearch_domains(domain_names: list.domain_names.map { |d| d.domain_name } )
    descs.domain_status_list.each { |domain|
      if args[:cloud_id]
        if [domain.arn, domain.domain_name, domain.domain_id].include?(args[:cloud_id])
          found[args[:cloud_id]] = domain
          return found
        end
      else
        found[domain.domain_name] = domain
      end
    }
  end

  # TODO consider a search by tags
  found
end
isGlobal?() click to toggle source

Does this resource type exist as a global (cloud-wide) artifact, or is it localized to a region/zone? @return [Boolean]

# File modules/mu/providers/aws/search_domain.rb, line 108
def self.isGlobal?
  false
end
new(**args) click to toggle source

Initialize this cloud resource object. Calling super will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us. @param args [Hash]: Hash of named arguments passed via Ruby's double-splat

Calls superclass method
# File modules/mu/providers/aws/search_domain.rb, line 23
def initialize(**args)
  super
  describe if @mu_name and !@deploydata
  @cloud_id ||= @deploydata['domain_name'] if @deploydata

  @mu_name ||= @deploy.getResourceName(@config["name"])
end
quality() click to toggle source

Denote whether this resource implementation is experiment, ready for testing, or ready for production use.

# File modules/mu/providers/aws/search_domain.rb, line 114
def self.quality
  MU::Cloud::RELEASE
end
schema(_config) click to toggle source

Cloud-specific configuration properties. @param _config [MU::Config]: The calling MU::Config object @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource

# File modules/mu/providers/aws/search_domain.rb, line 288
def self.schema(_config)
  toplevel_required = ["elasticsearch_version", "instance_type"]

  versions = begin
    MU::Cloud::AWS.elasticsearch.list_elasticsearch_versions.elasticsearch_versions
  rescue MuError
    ["7.4", "7.1", "6.8", "6.7", "6.5", "6.4", "6.3", "6.2", "6.0", "5.6"]
  end
  instance_types = begin
    MU::Cloud::AWS.elasticsearch.list_elasticsearch_instance_types(
      elasticsearch_version: "6.3"
    ).elasticsearch_instance_types
  rescue MuError
    ["c5.large.elasticsearch", "c5.xlarge.elasticsearch", "c5.2xlarge.elasticsearch", "c5.4xlarge.elasticsearch", "c5.9xlarge.elasticsearch", "c5.18xlarge.elasticsearch", "i3.large.elasticsearch", "i3.xlarge.elasticsearch", "i3.2xlarge.elasticsearch", "i3.4xlarge.elasticsearch", "i3.8xlarge.elasticsearch", "i3.16xlarge.elasticsearch", "m5.large.elasticsearch", "m5.xlarge.elasticsearch", "m5.2xlarge.elasticsearch", "m5.4xlarge.elasticsearch", "m5.12xlarge.elasticsearch", "r5.large.elasticsearch", "r5.xlarge.elasticsearch", "r5.2xlarge.elasticsearch", "r5.4xlarge.elasticsearch", "r5.12xlarge.elasticsearch", "t2.small.elasticsearch", "t2.medium.elasticsearch", "c4.large.elasticsearch", "c4.xlarge.elasticsearch", "c4.2xlarge.elasticsearch", "c4.4xlarge.elasticsearch", "c4.8xlarge.elasticsearch", "i2.xlarge.elasticsearch", "i2.2xlarge.elasticsearch", "m4.large.elasticsearch", "m4.xlarge.elasticsearch", "m4.2xlarge.elasticsearch", "m4.4xlarge.elasticsearch", "m4.10xlarge.elasticsearch", "r4.large.elasticsearch", "r4.xlarge.elasticsearch", "r4.2xlarge.elasticsearch", "r4.4xlarge.elasticsearch", "r4.8xlarge.elasticsearch", "r4.16xlarge.elasticsearch", "m3.medium.elasticsearch", "m3.large.elasticsearch", "m3.xlarge.elasticsearch", "m3.2xlarge.elasticsearch", "r3.large.elasticsearch", "r3.xlarge.elasticsearch", "r3.2xlarge.elasticsearch", "r3.4xlarge.elasticsearch", "r3.8xlarge.elasticsearch"]
  rescue Aws::ElasticsearchService::Errors::ValidationException
    # Some regions (GovCloud) lag
    MU::Cloud::AWS.elasticsearch.list_elasticsearch_instance_types(
      elasticsearch_version: "6.2"
    ).elasticsearch_instance_types
  end

  polschema = MU::Config::Role.schema["properties"]["policies"]
  polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema)

  schema = {
    "name" => {
      "type" => "string",
      "pattern" => '^[a-z][a-z0-9\-]+$'
    },
    "elasticsearch_version" => {
      "type" => "string",
      "default" => versions.first,
      "description" => "A supported ElasticSearch version for the region of this SearchDomain. Known versions from #{MU.myRegion}: "+versions.join(", ")
    },
    "instance_type" => {
      "type" => "string",
      "default" => instance_types.first,
      "description" => "A supported ElasticSearch instance type for the region of this SearchDomain. Known types from #{MU.myRegion}: "+instance_types.join(", ")+"."
    },
    "dedicated_masters" => {
      "type" => "integer",
      "default" => 0,
      "description" => "Separate, dedicated master node(s), over and above the search instances specified in instance_count."
    },
    "policies" => polschema,
    "access_policies" => {
      "type" => "object",
      "description" => "An IAM policy document for access to ElasticSearch (see {policies} for setting complex access policies with runtime dependencies). Our parser expects this to be defined inline like the rest of your YAML/JSON Basket of Kittens, not as raw JSON. For guidance on ElasticSearch IAM capabilities, see: https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-ac.html"
    },
    "master_instance_type" => {
      "type" => "string",
      "description" => "Instance type for dedicated master nodes, if any were requested. Will default to match instance_type."
    },
    "ebs_type" => {
      "type" => "string",
      "default" => "gp2",
      "description" => "Type of EBS storage to use for cluster nodes. If 'none' is specified, EBS storage will not be used, but this is only valid for certain instance types.",
      "enum" => ["standard", "gp2", "io1", "none"]
    },
    "ebs_iops" => {
      "type" => "integer",
      "description" => "Specifies the IOPD for a Provisioned IOPS EBS volume (SSD). Must specify ebs_type for this to take effect."
    },
    "ebs_size" => {
      "type" => "integer",
      "default" => 20,
      "description" => "Specifies the size (GB) of EBS storage. Must specify ebs_type for this to take effect."
    },
    "snapshot_hour" => {
      "type" => "integer",
      "default" => 23,
      "description" => "Clock hour (UTC) to begin daily snapshots"
    },
    "kms_encryption_key_id" => {
      "type" => "string",
      "description" => "If specified, will attempt to enable encryption at rest with this KMS Key ID"
    },
    "zone_aware" => {
      "type" => "boolean",
      "default" => false,
      "description" => "Spread search instances across Availability Zones to facilitate replica index sharding for greater resilience. Note that you also must use the native Elasticsearch API to create replica shards for your cluster. Zone awareness requires an even number of instances in the instance count."
    },
    "slow_logs" => {
      "type" => "string",
      "description" => "The ARN of a CloudWatch Log Group to which we we'll send slow index and search logs. If not specified, a log group will be generated."
    },
    "advanced_options" => {
      "type" => "object",
      "description" => "Key => Value strings pairs that pass certain configuration options to Elasticsearch. For a list of supported values, see https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-createupdatedomains.html#es-createdomain-configure-advanced-options",
    },
    "cognito" => {
      "type" => "object",
      "description" => "Options to specify the Cognito user and identity pools for Kibana authentication. For more information, see http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-cognito-auth.html",
      "required" => ["user_pool_id", "identity_pool_id"],
      "properties" => {
        "user_pool_id" => {
          "type" => "string",
          "description" => "Amazon Cognito user pool. Looks like 'us-east-1:69e2223c-2c74-42ca-9b27-1037fcb60b91'. See https://docs.aws.amazon.com/cognito/latest/developerguide/cognito-user-identity-pools.html"
        },
        "identity_pool_id" => {
          "type" => "string",
          "description" => "Amazon Cognito identity pool. Looks like 'us-east-1_eSwWA1VGY'. See https://docs.aws.amazon.com/cognito/latest/developerguide/identity-pools.html"
        },
        "role_arn" => {
          "type" => "string",
          "description" => "An IAM role that has the AmazonESCognitoAccess policy attached. If not specified, one will be generated automatically."
        }
      }
    }
  }
  [toplevel_required, schema]
end
validateConfig(dom, configurator) click to toggle source

Cloud-specific pre-processing of {MU::Config::BasketofKittens::search_domains}, bare and unvalidated. @param dom [Hash]: The resource to process and validate @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member @return [Boolean]: True if validation succeeded, False otherwise

# File modules/mu/providers/aws/search_domain.rb, line 405
def self.validateConfig(dom, configurator)
  ok = true
  versions = MU::Cloud::AWS.elasticsearch(region: dom['region']).list_elasticsearch_versions.elasticsearch_versions
  if !versions.include?(dom["elasticsearch_version"])
    MU.log "Invalid ElasticSearch version '#{dom["elasticsearch_version"]}' in SearchDomain '#{dom['name']}'", MU::ERR, details: versions
    ok = false
  else
    resp = MU::Cloud::AWS.elasticsearch(region: dom['region']).list_elasticsearch_instance_types(
      elasticsearch_version: dom["elasticsearch_version"]
    )
  
    if resp.nil? or resp.elasticsearch_instance_types.nil?
      MU.log "Failed to list valid ElasticSearch instance types in #{dom['region']}", MU::WARN
    end

    if !resp.elasticsearch_instance_types.include?(dom["instance_type"])
      MU.log "Invalid instance_type '#{dom["instance_type"]}' in SearchDomain '#{dom['name']}'", MU::ERR, details: resp.elasticsearch_instance_types
      ok = false
    end
  end

  if dom["dedicated_masters"] > 0 and dom["master_instance_type"].nil?
    dom["master_instance_type"] = dom["instance_type"]
    if dom["dedicated_masters"] != 3 and dom["dedicated_masters"] != 5
      MU.log "SearchDomain #{dom['name']}: You must choose either three or five dedicated master nodes", MU::ERR
      ok = false
    end
  end

  if dom["instance_count"] < 1
    MU.log "Must have at least one search node in SearchDomain '#{dom['name']}'", MU::ERR
    ok = false
  end

  if dom["ebs_iops"]
    MU.log "SearchDomain #{dom['name']} declared ebs_iops, setting volume type to io1", MU::NOTICE
    dom["ebs_type"] = "io1"
  end

  if dom["zone_aware"] and (dom["instance_count"] % 2) != 0
    MU.log "Must set an even number for instance_count when enabling Zone Awareness in SearchDomain '#{dom['name']}'", MU::ERR
    ok = false
  end

  if !dom["vpc"]
    MU.log "No VPC specified for SearchDomain '#{dom['name']},' endpoints will be public", MU::NOTICE
    if (dom['ingress_rules'] and dom['ingress_rules'].size > 0) or
       (dom['add_firewall_rules'] and dom['add_firewall_rules'].size > 0)
      MU.log "You must deploy SearchDomain '#{dom['name']}' into a VPC in order to use ingress_rules", MU::ERR
      ok = false
    end
  else
    if dom['ingress_rules']
      fwname = "searchdomain-#{dom['name']}"
      acl = {"name" => fwname, "rules" => dom['ingress_rules'], "region" => dom['region'], "optional_tags" => dom['optional_tags']}
      acl["tags"] = dom['tags'] if dom['tags'] && !dom['tags'].empty?
      acl["vpc"] = dom['vpc'].dup if dom['vpc']
      ok = false if !configurator.insertKitten(acl, "firewall_rules")
      dom["add_firewall_rules"] = [] if dom["add_firewall_rules"].nil?
      dom["add_firewall_rules"] << {"name" => fwname}
    end
  end

  if dom['snapshot_hour'] < 0 or dom['snapshot_hour'] > 23
    MU.log "Invalid snapshot_hour in SearchDomain '#{dom['name']}', must be in the range 0..23", MU::ERR
    ok = false
  end

  if dom['slow_logs']
    if configurator.haveLitterMate?(dom['slow_logs'], "log")
      MU::Config.addDependency(dom, dom['slow_logs'], "log")
    else
      log_group = MU::Cloud.resourceClass("AWS", "Log").find(cloud_id: dom['slow_logs'], region: dom['region']).values.first
      if !log_group
        MU.log "Specified slow_logs CloudWatch log group '#{dom['slow_logs']}' in SearchDomain '#{dom['name']}' doesn't appear to exist", MU::ERR
        ok = false
      else
        dom['slow_logs'] = log_group.arn
      end
    end
  else
    dom['slow_logs'] = dom['name']+"-slowlog"
    log_group = {
      "name" => dom['slow_logs'],
      "credentials" => dom['credentials']
    }
    ok = false if !configurator.insertKitten(log_group, "logs")
    MU::Config.addDependency(dom, dom['slow_logs'], "log")
  end

  if dom['advanced_options']
    dom['advanced_options'].each_pair { |key, val|
      dom['advanced_options'][key] = val.to_s
    }
  end

  if dom['cognito']
    begin
      MU::Cloud::AWS.cognito_ident(region: dom['region']).describe_identity_pool(
        identity_pool_id: dom['cognito']['identity_pool_id']
      )
    rescue ::Aws::CognitoIdentity::Errors::ValidationException, Aws::CognitoIdentity::Errors::ResourceNotFoundException
      MU.log "Cognito identity pool #{dom['cognito']['identity_pool_id']} malformed or does not exist in SearchDomain '#{dom['name']}'", MU::ERR
      ok = false
    end
    begin
      MU::Cloud::AWS.cognito_user(region: dom['region']).describe_user_pool(
        user_pool_id: dom['cognito']['user_pool_id']
      )
    rescue ::Aws::CognitoIdentityProvider::Errors::InvalidParameterException, Aws::CognitoIdentityProvider::Errors::ResourceNotFoundException
      MU.log "Cognito identity pool #{dom['cognito']['user_pool_id']} malformed or does not exist in SearchDomain '#{dom['name']}'", MU::ERR
      ok = false
    end

    if dom['cognito']['role_arn']
      rolename = dom['cognito']['role_arn'].sub(/.*?:role\/([a-z0-9-]+)$/, '\1')
      begin
        if !dom['cognito']['role_arn'].match(/^arn:/)
          role = MU::Cloud::AWS.iam.get_role(role_name: rolename)
          dom['cognito']['role_arn'] = role.role.arn
        end
        pols = MU::Cloud::AWS.iam.list_attached_role_policies(role_name: rolename).attached_policies
        found = false
        pols.each { |policy|
          found = true if policy.policy_name == "AmazonESCognitoAccess"
        }
        if !found
          MU.log "IAM role #{dom['cognito']['role_arn']} exists, but not does have the AmazonESCognitoAccess policy attached. SearchDomain '#{dom['name']}' may not have necessary Cognito permissions.", MU::WARN
        end
      rescue Aws::IAM::Errors::NoSuchEntity
        MU.log "IAM role #{dom['cognito']['role_arn']} malformed or does not exist in SearchDomain '#{dom['name']}'", MU::ERR
        ok = false
      end
    else
      roledesc = {
        "name" => dom['name']+"cognitorole",
        "credentials" => dom['credentials'],
        "can_assume" => [
          {
            "entity_id" => "es.amazonaws.com",
            "entity_type" => "service"
          }
        ],
        "import" => [
          "AmazonESCognitoAccess"
        ]
      }
      configurator.insertKitten(roledesc, "roles")
      MU::Config.addDependency(dom, dom['name']+"cognitorole", "role")
    end

  end

  # TODO queue['access_policies'] should generate a policy blob via MU::Cloud::AWS::Role

  ok
end

Public Instance Methods

arn() click to toggle source

Canonical Amazon Resource Number for this resource @return [String]

# File modules/mu/providers/aws/search_domain.rb, line 81
def arn
  return nil if !cloud_desc
  cloud_desc.arn.dup
end
cloud_desc(use_cache: true) click to toggle source

Wrapper for cloud_desc method that deals with finding the AWS domain_name parameter, which isn't what we'd call ourselves if we had our druthers.

# File modules/mu/providers/aws/search_domain.rb, line 66
def cloud_desc(use_cache: true)
  return @cloud_desc_cache if @cloud_desc_cache and use_cache
  @cloud_id ||= @config['domain_name']
  return nil if !@cloud_id
  MU.retrier([::Aws::ElasticsearchService::Errors::ResourceNotFoundException], wait: 10, max: 12) {
    @cloud_desc_cache = MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).describe_elasticsearch_domain(
      domain_name: @cloud_id
    ).domain_status
  }

  @cloud_desc_cache
end
create() click to toggle source

Called automatically by {MU::Deploy#createResources}

# File modules/mu/providers/aws/search_domain.rb, line 32
def create
  @config['domain_name'] = @deploy.getResourceName(@config["name"], max_length: 28, need_unique_string: true).downcase

  params = genParams

  MU.log "Creating ElasticSearch domain #{@config['domain_name']}", details: params
  @cloud_id = @config['domain_name']
  MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).create_elasticsearch_domain(params).domain_status

  tagDomain

end
groom() click to toggle source

Called automatically by {MU::Deploy#createResources}

# File modules/mu/providers/aws/search_domain.rb, line 46
def groom
  tagDomain
  @config['domain_name'] ||= @cloud_id
  params = genParams(cloud_desc) # get parameters that would change only

  if params.size > 1
    waitWhileProcessing # wait until the create finishes, if still going

    MU.log "Updating ElasticSearch domain #{@config['domain_name']}", MU::NOTICE, details: params
    MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).update_elasticsearch_domain_config(params)
  end

  waitWhileProcessing # don't return until creation/updating is complete
  MU.log "Search Domain #{@config['name']}: #{cloud_desc.endpoint}", MU::SUMMARY
end
notify() click to toggle source

Return the metadata for this SearchDomain rule @return [Hash]

# File modules/mu/providers/aws/search_domain.rb, line 88
def notify
  return nil if !cloud_desc(use_cache: false)
  deploy_struct = MU.structToHash(cloud_desc, stringify_keys: true)
  tags = MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).list_tags(arn: arn).tag_list
  deploy_struct['tags'] = tags.map { |t| { t.key => t.value } }
  if deploy_struct['endpoint']
    deploy_struct['kibana'] = deploy_struct['endpoint']+"/_plugin/kibana/"
  elsif deploy_struct['endpoints']
    deploy_struct['kibana'] = {}
    deploy_struct['endpoints'].each_pair { |k, v|
      deploy_struct['kibana'][k] = v+"/_plugin/kibana/"
    }
  end
  deploy_struct['domain_name'] ||= @config['domain_name'] if @config['domain_name']
  deploy_struct
end
toKitten(**_args) click to toggle source

Reverse-map our cloud description into a runnable config hash. We assume that any values we have in +@config+ are placeholders, and calculate our own accordingly based on what's live in the cloud.

# File modules/mu/providers/aws/search_domain.rb, line 198
def toKitten(**_args)
  bok = {
    "cloud" => "AWS",
    "credentials" => @credentials,
    "cloud_id" => @cloud_id,
    "region" => @region
  }

  if !cloud_desc
    MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
    return nil
  end

  bok['name'] = cloud_desc.domain_name
  bok['elasticsearch_version'] = cloud_desc.elasticsearch_version
  bok['instance_count'] = cloud_desc.elasticsearch_cluster_config.instance_count
  bok['instance_type'] = cloud_desc.elasticsearch_cluster_config.instance_type
  bok['zone_aware'] = cloud_desc.elasticsearch_cluster_config.zone_awareness_enabled

  if cloud_desc.elasticsearch_cluster_config.dedicated_master_enabled
    bok['dedicated_masters'] = cloud_desc.elasticsearch_cluster_config.dedicated_master_count
    bok['master_instance_type'] = cloud_desc.elasticsearch_cluster_config.dedicated_master_type
  end

  if cloud_desc.access_policies and !cloud_desc.access_policies.empty?
    bok['access_policies'] = JSON.parse(cloud_desc.access_policies)
  end

  if cloud_desc.advanced_options and !cloud_desc.advanced_options.empty?
    bok['advanced_options'] = cloud_desc.advanced_options
  end

  bok['ebs_size'] = cloud_desc.ebs_options.volume_size
  bok['ebs_type'] = cloud_desc.ebs_options.volume_type
  bok['ebs_iops'] = cloud_desc.ebs_options.iops if cloud_desc.ebs_options.iops

  if cloud_desc.snapshot_options and cloud_desc.snapshot_options.automated_snapshot_start_hour
    bok['snapshot_hour'] = cloud_desc.snapshot_options.automated_snapshot_start_hour
  end

  if cloud_desc.cognito_options.user_pool_id and
     cloud_desc.cognito_options.identity_pool_id
    bok['user_pool_id'] = cloud_desc.cognito_options.user_pool_id
    bok['identity_pool_id'] = cloud_desc.cognito_options.identity_pool_id
  end

  tags = MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).list_tags(arn: cloud_desc.arn).tag_list
  if tags and !tags.empty?
    bok['tags'] = MU.structToHash(tags)
  end

  if cloud_desc.vpc_options
    bok['vpc'] = MU::Config::Ref.get(
      id: cloud_desc.vpc_options.vpc_id,
      cloud: "AWS",
      credentials: @credentials,
      type: "vpcs",
      region: @region,
      subnets: cloud_desc.vpc_options.subnet_ids.map { |s| { "subnet_id" => s } }
    )
    if cloud_desc.vpc_options.security_group_ids and
       !cloud_desc.vpc_options.security_group_ids.empty?
      bok['add_firewall_rules'] = cloud_desc.vpc_options.security_group_ids.map { |sg|
        MU::Config::Ref.get(
          id: sg,
          cloud: "AWS",
          credentials: @credentials,
          region: @region,
          type: "firewall_rules",
        )
      }
    end
  end

  if cloud_desc.log_publishing_options
    # XXX this is primitive... there are multiple other log types now,
    # and this should be a Ref blob, not a flat string
    cloud_desc.log_publishing_options.each_pair { |type, whither|
      if type == "SEARCH_SLOW_LOGS"
        bok['slow_logs'] = whither.cloud_watch_logs_log_group_arn
      end
    }
  end

  bok
end

Private Instance Methods

genParams(ext = nil) click to toggle source

create_elasticsearch_domain and update_elasticsearch_domain_config take almost the same set of parameters, so our create and groom methods do nearly the same things. Factor it. If we're operating on an existing domain, only return things that would be changed.

# File modules/mu/providers/aws/search_domain.rb, line 569
def genParams(ext = nil)
  params = {
    :domain_name => @config['domain_name'] || @deploydata['domain_name']
  }

  if ext.nil?
    params[:elasticsearch_version] = @config['elasticsearch_version']
  elsif ext.elasticsearch_version != @config['elasticsearch_version']

    raise MU::MuError, "Can't change ElasticSearch version of an existing cluster"
  end

  if ext.nil? or
     ext.elasticsearch_cluster_config.instance_type != @config['instance_type'] or
     ext.elasticsearch_cluster_config.instance_count != @config['instance_count'] or
     ext.elasticsearch_cluster_config.zone_awareness_enabled != @config['zone_aware']
    params[:elasticsearch_cluster_config] = {}
    params[:elasticsearch_cluster_config][:instance_type] = @config['instance_type']
    params[:elasticsearch_cluster_config][:instance_count] = @config['instance_count']
    params[:elasticsearch_cluster_config][:zone_awareness_enabled] = @config['zone_aware']
  end

  if @config['dedicated_masters'] > 0
    if ext.nil? or !ext.elasticsearch_cluster_config.dedicated_master_enabled or
       ext.elasticsearch_cluster_config.dedicated_master_count != @config['dedicated_masters'] or
       ext.elasticsearch_cluster_config.dedicated_master_type != @config['master_instance_type']
      params[:elasticsearch_cluster_config][:dedicated_master_enabled] = true
      params[:elasticsearch_cluster_config][:dedicated_master_count] = @config['dedicated_masters']
      params[:elasticsearch_cluster_config][:dedicated_master_type] = @config['master_instance_type']
    end
  end

  if ext.nil? or ext.snapshot_options.automated_snapshot_start_hour != @config['snapshot_hour']
    params[:snapshot_options] = {}
    params[:snapshot_options][:automated_snapshot_start_hour] = @config['snapshot_hour']
  end

  if ext
    # Despite being called access_policies, this parameter actually
    # only accepts one policy. So, we'll munge everything we have
    # together into one policy with multiple Statements.
    policy = nil
    # TODO check against ext.access_policy.options

    if @config['access_policies']
      policy = @config['access_policies']
      # ensure the "Statement" key is cased in a predictable way
      statement_key = nil
      policy.each_pair { |k, v|
        if k.downcase == "statement" and k != "Statement"
          statement_key = k
          break
        end
      }
      if statement_key
        policy["Statement"] = policy.delete(statement_key)
      end
      if !policy["Statement"].is_a?(Array)
        policy["Statement"] = [policy["Statement"]]
      end
    end

    if @config['policies']
      @config['policies'].each { |p|
        p['targets'].each { |t|
          if t['path']
            t['path'].gsub!(/#SELF/, @mu_name.downcase)
          end
        }
        parsed = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument([p], deploy_obj: @deploy, bucket_style: true).first.values.first

        if policy and policy["Statement"]
          policy["Statement"].concat(parsed["Statement"])
        else
          policy = parsed
        end
      }
    end

    if policy
      params[:access_policies] = JSON.generate(policy)
    end
  end

  if @config['slow_logs']
    arn = nil
    if @config['slow_logs'].match(/^arn:/i)
      arn = @config['slow_logs']
    else
      log_group = @deploy.findLitterMate(type: "log", name: @config['slow_logs'])
      log_group = MU::Cloud.resourceClass("AWS", "Log").find(cloud_id: log_group.mu_name, region: log_group.cloudobj.config['region']).values.first
      if log_group.nil? or log_group.arn.nil?
        raise MuError, "Failed to retrieve ARN of sibling LogGroup '#{@config['slow_logs']}'"
      end
      arn = log_group.arn
    end

    if arn
      @config['slow_logs'] = arn
    end

    if ext.nil? or
        ext.log_publishing_options.nil? or
        ext.log_publishing_options["INDEX_SLOW_LOGS"].nil? or
        !ext.log_publishing_options["INDEX_SLOW_LOGS"][:enabled] or
        ext.log_publishing_options["INDEX_SLOW_LOGS"][:cloud_watch_logs_log_group_arn] != arn or
        ext.log_publishing_options["SEARCH_SLOW_LOGS"].nil? or
        !ext.log_publishing_options["SEARCH_SLOW_LOGS"][:enabled] or
        ext.log_publishing_options["SEARCH_SLOW_LOGS"][:cloud_watch_logs_log_group_arn] != arn
      params[:log_publishing_options] = {}
      params[:log_publishing_options]["INDEX_SLOW_LOGS"] = {}
      params[:log_publishing_options]["INDEX_SLOW_LOGS"][:enabled] = true
      params[:log_publishing_options]["INDEX_SLOW_LOGS"][:cloud_watch_logs_log_group_arn] = arn

      params[:log_publishing_options]["SEARCH_SLOW_LOGS"] = {}
      params[:log_publishing_options]["SEARCH_SLOW_LOGS"][:enabled] = true
      params[:log_publishing_options]["SEARCH_SLOW_LOGS"][:cloud_watch_logs_log_group_arn] = arn
      MU::Cloud.resourceClass("AWS", "Log").allowService("es.amazonaws.com", arn, @region)
    end
  end

  if @config['advanced_options'] and (ext.nil? or 
     ext.advanced_options != @config['advanced_options'])
    params[:advanced_options] = {}
    @config['advanced_options'].each_pair { |key, value|
      params[:advanced_options][key] = value
    }
  end

  if @config['vpc']
    subnet_ids = []
    sgs = []
    if !@config["vpc"]["subnets"].nil? and @config["vpc"]["subnets"].size > 0
      @config["vpc"]["subnets"].each { |subnet|
        subnet_obj = @vpc.getSubnet(cloud_id: subnet["subnet_id"], name: subnet["subnet_name"])
        subnet_ids << subnet_obj.cloud_id
      }
    else
      @vpc.subnets.each { |subnet_obj|
        next if subnet_obj.private? and ["all_public", "public"].include?(@config["vpc"]["subnet_pref"])
        next if !subnet_obj.private? and ["all_private", "private"].include?(@config["vpc"]["subnet_pref"])
        subnet_ids << subnet_obj.cloud_id
      }
    end
    if subnet_ids.size == 0
      raise MuError, "No valid subnets found for #{@mu_name} from #{@config["vpc"]}"
    end

    if @dependencies.has_key?("firewall_rule")
      @dependencies['firewall_rule'].values.each { |sg|
        sgs << sg.cloud_id
      }
    end

    # XXX this will break on regroom, revisit and make deterministic
    # or remembered
    subnet_ids = subnet_ids.sample(3) if subnet_ids.size > 3

    if ext.nil? or
       ext.vpc_options.subnet_ids != subnet_ids or
       ext.vpc_options.security_group_ids != sgs
      params[:vpc_options] = {}
      params[:vpc_options][:subnet_ids] = subnet_ids
      params[:vpc_options][:security_group_ids] = sgs
    end
    if @config['zone_aware'] and params[:elasticsearch_cluster_config]
      params[:elasticsearch_cluster_config][:zone_awareness_config] = {
        :availability_zone_count => subnet_ids.size
      }
    end
  end

  if @config['ebs_type']
    if ext.nil? or ext.ebs_options.nil? or !ext.ebs_options.ebs_enabled or
       ext.ebs_options.volume_type != @config['ebs_type'] or
       ext.ebs_options.volume_size != @config['ebs_size'] or
       ext.ebs_options.iops != @config['ebs_iops']
      params[:ebs_options] = {}
      params[:ebs_options][:ebs_enabled] = true
      params[:ebs_options][:volume_type] = @config['ebs_type']
      params[:ebs_options][:volume_size] = @config['ebs_size']
      if @config['ebs_iops']
        params[:ebs_options][:iops] = @config['ebs_iops']
      end
    end
  end

  if @config['kms_encryption_key_id']
    if ext.nil? or !ext.encryption_at_rest_options.enabled or
       ext.kms_key_id != @config['kms_encryption_key_id']
      params[:encryption_at_rest_options] = {}
      params[:encryption_at_rest_options][:enabled] = true
      params[:encryption_at_rest_options][:kms_key_id] = @config['kms_encryption_key_id']
    end
  end


  # XXX API fails with "Amazon Elasticsearch must be allowed to use the
  # passed role" when we do this on creation, but it works fine if we
  # modify an existing group. AWS bug, workaround is to just apply
  # this in groom phase exclusively.
  if @config['cognito'] and !ext.nil?
    setIAMPolicies

    if ext.nil? or !ext.cognito_options.enabled or
       ext.cognito_options.user_pool_id != @config['cognito']['user_pool_id'] or
       ext.cognito_options.identity_pool_id != @config['cognito']['identity_pool_id'] or
       (@config['cognito']['role_arn'] and ext.cognito_options.role_arn != @config['cognito']['role_arn'])
      params[:cognito_options] = {}
      params[:cognito_options][:enabled] = true
      params[:cognito_options][:user_pool_id] = @config['cognito']['user_pool_id']
      params[:cognito_options][:identity_pool_id] = @config['cognito']['identity_pool_id']
      if @config['cognito']['role_arn']
        params[:cognito_options][:role_arn] = @config['cognito']['role_arn']
      else
        myrole = @deploy.findLitterMate(name: @config['name']+"cognitorole", type: "roles")
        params[:cognito_options][:role_arn] = myrole.cloudobj.arn
      end
    end
  end

  params
end
tagDomain() click to toggle source
# File modules/mu/providers/aws/search_domain.rb, line 793
def tagDomain
  tags = [{ key: "Name", value: @mu_name }]

  MU::MommaCat.listStandardTags.each_pair { |name, value|
    tags << {key: name, value: value }
  }

  if @config['optional_tags']
    MU::MommaCat.listOptionalTags.each_pair { |name, value|
      tags << {key: name, value: value }
    }
  end

  if @config['tags']
    @config['tags'].each { |tag|
      tags << {key: tag['key'], value: tag['value'] }
    }
  end
  domain = cloud_desc
  if !domain or !domain.arn
    raise MU::MuError, "Can't tag ElasticSearch domain, cloud descriptor came back without an ARN"
  end

  MU::Cloud::AWS.elasticsearch(region: @region, credentials: @credentials).add_tags(
    arn: domain.arn,
    tag_list: tags
  )
end
waitWhileProcessing() click to toggle source
# File modules/mu/providers/aws/search_domain.rb, line 822
def waitWhileProcessing
  retries = 0
  interval = 60

  begin
    resp = cloud_desc(use_cache: false)

    if (resp.endpoint.nil? or resp.endpoint.empty?) and
       (resp.endpoints.nil? or resp.endpoints.empty?) and
       !resp.deleted
      loglevel = (retries > 0 and retries % 3 == 0) ? MU::NOTICE : MU::DEBUG
      MU.log "Waiting for Elasticsearch domain #{@mu_name} (#{@config['domain_name']}) to finish creating", loglevel
      sleep interval
    end
    retries += 1
  end while (resp.endpoint.nil? or resp.endpoint.empty?) and (resp.endpoints.nil? or resp.endpoints.empty?) and !resp.deleted
end