class MU::Cloud::Google

Support for Google Cloud Platform as a provisioning layer.

Attributes

customer[R]
project_id[R]

Public Class Methods

adminBucketName(credentials = nil) click to toggle source

Resolve the administrative Cloud Storage bucket for a given credential set, or return a default. @param credentials [String] @return [String]

# File modules/mu/providers/google.rb, line 265
def self.adminBucketName(credentials = nil)
   #XXX find a default if this particular account doesn't have a log_bucket_name configured
  cfg = credConfig(credentials)
  if cfg.nil?
    raise MuError, "Failed to load Google credential set #{credentials}"
  end
  cfg['log_bucket_name']
end
adminBucketUrl(credentials = nil) click to toggle source

Resolve the administrative Cloud Storage bucket for a given credential set, or return a default. @param credentials [String] @return [String]

# File modules/mu/providers/google.rb, line 278
def self.adminBucketUrl(credentials = nil)
  "gs://"+adminBucketName(credentials)+"/"
end
admin_directory(subclass = nil, credentials: nil) click to toggle source

GCP's AdminDirectory Service API @param subclass [<Google::Apis::AdminDirectoryV1>]: If specified, will return the class ::Google::Apis::AdminDirectoryV1::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 859
def self.admin_directory(subclass = nil, credentials: nil)
  require 'google/apis/admin_directory_v1'

  # fill in the default credential set name so we don't generate
  # dopey extra warnings about falling back on scopes
  credentials ||= MU::Cloud::Google.credConfig(credentials, name_only: true)

  writescopes = ['admin.directory.group.member', 'admin.directory.group', 'admin.directory.user', 'admin.directory.domain', 'admin.directory.orgunit', 'admin.directory.rolemanagement', 'admin.directory.customer', 'admin.directory.user.alias', 'admin.directory.userschema']
  readscopes = ['admin.directory.group.member.readonly', 'admin.directory.group.readonly', 'admin.directory.user.readonly', 'admin.directory.domain.readonly', 'admin.directory.orgunit.readonly', 'admin.directory.rolemanagement.readonly', 'admin.directory.customer.readonly', 'admin.directory.user.alias.readonly', 'admin.directory.userschema.readonly']
  @@readonly_semaphore.synchronize {
    use_scopes = readscopes+writescopes
    if @@readonly[credentials] and @@readonly[credentials]["AdminDirectoryV1"]
      use_scopes = readscopes.dup
    end

    if subclass.nil?
      begin
        @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: use_scopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials, auth_error_quiet: true)
      rescue Signet::AuthorizationError
        MU.log "Falling back to read-only access to DirectoryService API for credential set '#{credentials}'", MU::WARN
        @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: readscopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
        @@readonly[credentials] ||= {}
        @@readonly[credentials]["AdminDirectoryV1"] = true
      end
      return @@admin_directory_api[credentials]
    elsif subclass.is_a?(Symbol)
      return Object.const_get("::Google").const_get("Apis").const_get("AdminDirectoryV1").const_get(subclass)
    end
  }
end
billing(subclass = nil, credentials: nil) click to toggle source

Google's Cloud Billing Budget Service API @param subclass [<Google::Apis::CloudbillingV1>]: If specified, will return the class ::Google::Apis::CloudbillingV1::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 999
def self.billing(subclass = nil, credentials: nil)
  require 'google/apis/cloudbilling_v1'

  if subclass.nil?
    @@billing_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudbillingV1::CloudbillingService", scopes: ['cloud-platform', 'cloud-billing'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
    return @@billing_api[credentials]
  elsif subclass.is_a?(Symbol)
    return Object.const_get("::Google").const_get("Apis").const_get("CloudbillingV1").const_get(subclass)
  end
end
budgets(subclass = nil, credentials: nil) click to toggle source

Google's Cloud Billing Service API @param subclass [<Google::Apis::CloudbillingV1>]: If specified, will return the class ::Google::Apis::CloudbillingV1::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 986
def self.budgets(subclass = nil, credentials: nil)
  require 'google/apis/billingbudgets_v1'

  if subclass.nil?
    @@budgets_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "BillingbudgetsV1::CloudBillingBudgetService", scopes: ['cloud-platform', 'cloud-billing'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
    return @@budgets_api[credentials]
  elsif subclass.is_a?(Symbol)
    return Object.const_get("::Google").const_get("Apis").const_get("BillingbudgetsV1").const_get(subclass)
  end
end
cleanDeploy(deploy_id, credentials: nil, noop: false) click to toggle source

Purge cloud-specific deploy meta-artifacts (SSH keys, resource groups, etc) @param deploy_id [MU::MommaCat]

# File modules/mu/providers/google.rb, line 344
def self.cleanDeploy(deploy_id, credentials: nil, noop: false)
  removeDeploySecretsAndRoles(deploy_id, noop: noop, credentials: credentials)
end
compute(subclass = nil, credentials: nil) click to toggle source

Google's Compute Service API @param subclass [<Google::Apis::ComputeV1>]: If specified, will return the class ::Google::Apis::ComputeV1::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 820
def self.compute(subclass = nil, credentials: nil)
  require 'google/apis/compute_v1'

  if subclass.nil?
    @@compute_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "ComputeV1::ComputeService", scopes: ['cloud-platform', 'compute.readonly'], credentials: credentials)
    return @@compute_api[credentials]
  elsif subclass.is_a?(Symbol)
    return Object.const_get("::Google").const_get("Apis").const_get("ComputeV1").const_get(subclass)
  end
end
config_example() click to toggle source

A non-working example configuration

# File modules/mu/providers/google.rb, line 118
def self.config_example
  sample = hosted_config
  sample ||= {
    "project" => "my-project",
    "region" => "us-east4"
  }
  sample["credentials_file"] = "#{Etc.getpwuid(Process.uid).dir}/gcp_serviceacct.json"
  sample["log_bucket_name"] = "my-mu-cloud-storage-bucket"
  sample
end
container(subclass = nil, credentials: nil) click to toggle source

Google's Container API @param subclass [<Google::Apis::ContainerV1>]: If specified, will return the class ::Google::Apis::ContainerV1::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 921
def self.container(subclass = nil, credentials: nil)
  require 'google/apis/container_v1'

  if subclass.nil?
    @@container_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "ContainerV1::ContainerService", scopes: ['cloud-platform'], credentials: credentials)
    return @@container_api[credentials]
  elsif subclass.is_a?(Symbol)
    return Object.const_get("::Google").const_get("Apis").const_get("ContainerV1").const_get(subclass)
  end
end
createSSLCertificate(name, cert, key, flags = {}, credentials: nil) click to toggle source

Create an SSL Certificate resource from some local x509 cert files. @param name [String]: A resource name for the certificate @param cert [String,OpenSSL::X509::Certificate]: An x509 certificate @param key [String,OpenSSL::PKey]: An x509 private key @return [Google::Apis::ComputeV1::SslCertificate]

# File modules/mu/providers/google.rb, line 511
def self.createSSLCertificate(name, cert, key, flags = {}, credentials: nil)
  flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
  flags["description"] ||= MU.deploy_id
  certobj = ::Google::Apis::ComputeV1::SslCertificate.new(
    name: name,
    certificate: cert.to_s,
    private_key: key.to_s,
    description: flags["description"]
  )
  MU::Cloud::Google.compute(credentials: credentials).insert_ssl_certificate(flags["project"], certobj)
end
credConfig(name = nil, name_only: false) click to toggle source

Return the $MU_CFG data associated with a particular profile/name/set of credentials. If no account name is specified, will return one flagged as default. Returns nil if GCP is not configured. Throws an exception if an account name is specified which does not exist. @param name [String]: The name of the key under 'google' in mu.yaml to return @return [Hash,nil]

# File modules/mu/providers/google.rb, line 288
      def self.credConfig(name = nil, name_only: false)
        # If there's nothing in mu.yaml (which is wrong), but we're running
        # on a machine hosted in GCP, fake it with that machine's service
        # account and hope for the best.
        if !$MU_CFG['google'] or !$MU_CFG['google'].is_a?(Hash) or $MU_CFG['google'].size == 0
          return @@my_hosted_cfg if @@my_hosted_cfg

          if hosted?
            @@my_hosted_cfg = hosted_config
            return name_only ? "#default" : @@my_hosted_cfg
          end

          return nil
        end

        if name.nil?
          $MU_CFG['google'].each_pair { |set, cfg|
            if cfg['default']
              return name_only ? set : cfg
            end
          }
        else
          if $MU_CFG['google'][name]
            return name_only ? name : $MU_CFG['google'][name]
          elsif @@acct_to_profile_map[name.to_s]
            return name_only ? name : @@acct_to_profile_map[name.to_s]
          end
# XXX whatever process might lead us to populate @@acct_to_profile_map with some mappings, like projectname -> account profile, goes here
          return nil
        end
      end
customerID(credentials = nil) click to toggle source

Fetch the GSuite/Cloud Identity customer id for the domain associated with the given credentials, if a domain is set via the masquerade_as configuration option.

# File modules/mu/providers/google.rb, line 1082
def self.customerID(credentials = nil)
  cfg = credConfig(credentials)
  if !cfg or !cfg['masquerade_as']
    return nil
  end

  if @@customer_ids_cache[credentials]
    return @@customer_ids_cache[credentials]
  end

  user = MU::Cloud::Google.admin_directory(credentials: credentials).get_user(cfg['masquerade_as'])
  if user and user.customer_id
    @@customer_ids_cache[credentials] = user.customer_id
  end

  @@customer_ids_cache[credentials]
end
defaultFolder(credentials = nil) click to toggle source

We want a default place to put new projects for the Habitat resource, so if we have a root folder, we can go ahead and use that. @param credentials [String] @return [String]

# File modules/mu/providers/google.rb, line 699
def self.defaultFolder(credentials = nil)
  project = defaultProject(credentials)
  resp = MU::Cloud::Google.resource_manager(credentials: credentials).get_project_ancestry(project)
  resp.ancestor.each { |a|
    if a.resource_id.type == "folder"
      return a.resource_id.id
    end
  }
  nil
end
defaultProject(credentials = nil) click to toggle source

Our credentials map to a project, an organizational structure in Google Cloud. This fetches the identifier of the project associated with our default credentials. @param credentials [String] @return [String]

# File modules/mu/providers/google.rb, line 666
def self.defaultProject(credentials = nil)
  if @@default_project_cache.has_key?(credentials)
    return @@default_project_cache[credentials]
  end
  cfg = credConfig(credentials)
  if !cfg or !cfg['project']
    if hosted?
      @@default_project_cache[credentials] = myProject
      return myProject 
    end
    if cfg
      begin
        result = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects
        result.projects.reject! { |p| p.lifecycle_state == "DELETE_REQUESTED" }
        available = result.projects.map { |p| p.project_id }
        if available.size == 1
          @@default_project_cache[credentials] = available[0]
          return available[0]
        end
      rescue # fine
      end
    end
  end
  return nil if !cfg
  loadCredentials(credentials) if !@@authorizers[credentials]
  @@default_project_cache[credentials] = cfg['project']
  cfg['project']
end
findLocationArgs(**args) click to toggle source

Most of our resource implementation find methods have to mangle their args to make sure they've extracted a project or location argument from other available information. This does it for them. @return [Hash]

# File modules/mu/providers/google.rb, line 64
def self.findLocationArgs(**args)
  args[:project] ||= args[:habitat]
  args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
  args[:location] ||= args[:region] || args[:availability_zone] || "-"
  args
end
firestore(subclass = nil, credentials: nil) click to toggle source

Google's Firestore (NoSQL) Service API @param subclass [<Google::Apis::FirestoreV1>]: If specified, will return the class ::Google::Apis::FirestoreV1::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 960
def self.firestore(subclass = nil, credentials: nil)
  require 'google/apis/firestore_v1'

  if subclass.nil?
    @@firestore_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "FirestoreV1::FirestoreService", scopes: ['cloud-platform'], credentials: credentials)
    return @@firestore_api[credentials]
  elsif subclass.is_a?(Symbol)
    return Object.const_get("::Google").const_get("Apis").const_get("FirestoreV1").const_get(subclass)
  end
end
folder(subclass = nil, credentials: nil) click to toggle source

Google's Cloud Resource Manager API V2, which apparently has all the folder bits @param subclass [<Google::Apis::CloudresourcemanagerV2>]: If specified, will return the class ::Google::Apis::CloudresourcemanagerV2::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 908
def self.folder(subclass = nil, credentials: nil)
  require 'google/apis/cloudresourcemanager_v2'

  if subclass.nil?
    @@resource2_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV2::CloudResourceManagerService", scopes: ['cloud-platform', 'cloudplatformfolders'], credentials: credentials,  masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
    return @@resource2_api[credentials]
  elsif subclass.is_a?(Symbol)
    return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV2").const_get(subclass)
  end
end
function(subclass = nil, credentials: nil) click to toggle source

Google's Cloud Function Service API @param subclass [<Google::Apis::CloudfunctionsV1>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 1012
def self.function(subclass = nil, credentials: nil)
  require 'google/apis/cloudfunctions_v1'

  if subclass.nil?
    @@function_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudfunctionsV1::CloudFunctionsService", scopes: ['cloud-platform'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
    return @@function_api[credentials]
  elsif subclass.is_a?(Symbol)
    return Object.const_get("::Google").const_get("Apis").const_get("CloudfunctionsV1").const_get(subclass)
  end
end
get(url) click to toggle source

Fetch a URL

# File modules/mu/providers/google.rb, line 625
def self.get(url)
  uri = URI url
  resp = nil

  Net::HTTP.start(uri.host, uri.port) do |http|
    resp = http.get(uri)
  end

  unless resp.code == "200"
    puts resp.code, resp.body
    exit
  end
  resp.body
end
getDomains(credentials = nil) click to toggle source

Retrieve the domains, if any, which these credentials can manage via GSuite or Cloud Identity. @param credentials [String] @return [Array<String>],nil]

# File modules/mu/providers/google.rb, line 1027
def self.getDomains(credentials = nil)
  my_org = getOrg(credentials)
  return nil if !my_org

  resp = MU::Cloud::Google.admin_directory(credentials: credentials).list_domains(MU::Cloud::Google.customerID(credentials))
  resp.domains.map { |d| d.domain_name.downcase }
end
getGoogleMetaData(param) click to toggle source

Fetch a Google instance metadata parameter (example: instance/id). @param param [String]: The parameter name to fetch @return [String, nil]

# File modules/mu/providers/google.rb, line 487
def self.getGoogleMetaData(param)
  base_url = "http://metadata.google.internal/computeMetadata/v1"
  begin
    Timeout.timeout(2) do
      response = URI.open(
        "#{base_url}/#{param}",
        "Metadata-Flavor" => "Google"
      ).read
      return response
    end
  rescue Net::HTTPServerException, OpenURI::HTTPError, Timeout::Error, SocketError, Errno::EHOSTUNREACH, Errno::ENETUNREACH => e
    # This is fairly normal, just handle it gracefully
    logger = MU::Logger.new
    logger.log "Failed metadata request #{base_url}/#{param}: #{e.inspect}", MU::DEBUG
  end

  nil
end
getOrg(credentials = nil, with_id: nil) click to toggle source

Retrieve the organization, if any, to which these credentials belong. @param credentials [String] @return [Array<OpenStruct>],nil]

# File modules/mu/providers/google.rb, line 1039
def self.getOrg(credentials = nil, with_id: nil)
  creds = MU::Cloud::Google.credConfig(credentials)
  return nil if !creds
  credname = if creds and creds['name']
    creds['name']
  else
    "default"
  end 

  with_id ||= creds['org'] if creds['org'] 
  return @@orgmap[credname] if @@orgmap.has_key?(credname)
  resp = MU::Cloud::Google.resource_manager(credentials: credname).search_organizations

  if resp and resp.organizations
    # XXX no idea if it's possible to be a member of multiple orgs
    if !with_id
      @@orgmap[credname] = resp.organizations.first
      return resp.organizations.first
    else
      resp.organizations.each { |org|
        if org.name == with_id or org.display_name == with_id or
           org.name == "organizations/#{with_id}"
          @@orgmap[credname] = org
          return org
        end
      }
      return nil
    end
  end

  @@orgmap[credname] = nil

  
  MU.log "Unable to list_organizations with credentials #{credname}. If this account is part of a GSuite or Cloud Identity domain, verify that Oauth delegation is properly configured and that 'masquerade_as' is properly set for the #{credname} Google credential set in mu.yaml.", MU::ERR, details: ["https://cloud.google.com/resource-manager/docs/creating-managing-organization", "https://admin.google.com/AdminHome?chromeless=1#OGX:ManageOauthClients"]

  nil
end
get_machine_credentials(scopes, credentials = nil) click to toggle source
# File modules/mu/providers/google.rb, line 548
def self.get_machine_credentials(scopes, credentials = nil)
  @@svc_account_name = MU::Cloud::Google.getGoogleMetaData("instance/service-accounts/default/email")
  MU.log "We are hosted in GCP, so I will attempt to use the service account #{@@svc_account_name} to make API requests.", MU::DEBUG

  @@authorizers[credentials][scopes.to_s] = ::Google::Auth.get_application_default(scopes)
  @@authorizers[credentials][scopes.to_s].fetch_access_token!
  @@default_project ||= MU::Cloud::Google.getGoogleMetaData("project/project-id")
  begin
    listRegions(credentials: credentials)
    listInstanceTypes(credentials: credentials)
    listHabitats(credentials)
  rescue ::Google::Apis::ClientError
    MU.log "Found machine credentials #{@@svc_account_name}, but these don't appear to have sufficient permissions or scopes", MU::WARN, details: scopes
    @@authorizers.delete(credentials)
    return nil
  end
  @@authorizers[credentials][scopes.to_s]
end
grantDeploySecretAccess(acct, deploy_id = MU.deploy_id, name = nil, credentials: nil) click to toggle source

Grant access to appropriate Cloud Storage objects in our log/secret bucket for a deploy member. @param acct [String]: The service account (by email addr) to which we'll grant access @param deploy_id [String]: The deploy for which we're granting the secret XXX add equivalent for AWS and call agnostically

# File modules/mu/providers/google.rb, line 404
      def self.grantDeploySecretAccess(acct, deploy_id = MU.deploy_id, name = nil, credentials: nil)
        name ||= deploy_id+"-secret"
        aclobj = nil

        retries = 0
        begin
          MU.log "Granting #{acct} access to list Cloud Storage bucket #{adminBucketName(credentials)}"
          MU::Cloud::Google.storage(credentials: credentials).insert_bucket_access_control(
            adminBucketName(credentials),
            MU::Cloud::Google.storage(:BucketAccessControl).new(
              bucket: adminBucketName(credentials),
              role: "READER",
              entity: "user-"+acct
            )
          )

          aclobj = MU::Cloud::Google.storage(:ObjectAccessControl).new(
            bucket: adminBucketName(credentials),
            role: "READER",
            entity: "user-"+acct
          )

          [name].each { |obj|
            MU.log "Granting #{acct} access to #{obj} in Cloud Storage bucket #{adminBucketName(credentials)}"

            MU::Cloud::Google.storage(credentials: credentials).insert_object_access_control(
              adminBucketName(credentials),
              obj,
              aclobj
            )
          }
        rescue ::Google::Apis::ClientError => e
MU.log e.message, MU::WARN, details: e.inspect
          if e.inspect.match(/body: "Not Found"/)
            raise MuError, "Google admin bucket #{adminBucketName(credentials)} or key #{name} does not appear to exist or is not visible with #{credentials ? credentials : "default"} credentials"
          elsif e.message.match(/notFound: |Unknown user:|conflict: /)
            if retries < 5
              sleep 5
              retries += 1
              retry
            else
              raise e
            end
          elsif e.inspect.match(/The metadata for object "null" was edited during the operation/)
            MU.log e.message+" - Google admin bucket #{adminBucketName(credentials)}/#{name} with #{credentials ? credentials : "default"} credentials", MU::DEBUG, details: aclobj
            sleep 10
            retry
          else
            raise MuError, "Got #{e.message} trying to set ACLs for #{deploy_id} in #{adminBucketName(credentials)}"
          end
        end
      end
habitat(cloudobj, nolookup: false, deploy: nil) click to toggle source

Return what we think of as a cloud object's habitat. In GCP, this means the project_id in which is resident. If this is not applicable, such as for a {Habitat} or {Folder}, returns nil. @param cloudobj [MU::Cloud::Google]: The resource from which to extract the habitat id @return [String,nil]

# File modules/mu/providers/google.rb, line 157
      def self.habitat(cloudobj, nolookup: false, deploy: nil)
        @@habmap ||= {}
# XXX whaddabout config['habitat'] HNNNGH
        return nil if !cloudobj.cloudclass.canLiveIn.include?(:Habitat)

# XXX these are assholes because they're valid two different ways ugh ugh
        return nil if [MU::Cloud::Google::Group, MU::Cloud::Google::Folder].include?(cloudobj.cloudclass)
        if cloudobj.config and cloudobj.config['project']
          if nolookup
            return cloudobj.config['project']
          end
          if @@habmap[cloudobj.config['project']]
            return @@habmap[cloudobj.config['project']]
          end
          deploy ||= cloudobj.deploy if cloudobj.respond_to?(:deploy)
          projectobj = projectLookup(cloudobj.config['project'], deploy, raise_on_fail: false)

          if projectobj
            @@habmap[cloudobj.config['project']] = projectobj.cloud_id
            return projectobj.cloud_id
          end
        end

        # blow up if this resource *has* to live in a project
        if cloudobj.cloudclass.canLiveIn == [:Habitat]
          MU.log "Failed to find project for cloudobj #{cloudobj.to_s}", MU::ERR, details: cloudobj
          raise MuError, "Failed to find project for cloudobj #{cloudobj.to_s}"
        end

        nil
      end
hosted() click to toggle source

Alias for #{MU::Cloud::Google.hosted?}

# File modules/mu/providers/google.rb, line 460
def self.hosted
  MU::Cloud::Google.hosted?
end
hosted?() click to toggle source

Determine whether we (the Mu master, presumably) are hosted in this cloud. @return [Boolean]

# File modules/mu/providers/google.rb, line 467
def self.hosted?
  if $MU_CFG.has_key?("google_is_hosted")
    @@is_in_aws = $MU_CFG["google_is_hosted"]
    return $MU_CFG["google_is_hosted"]
  end
  if !@@is_in_gcp.nil?
    return @@is_in_gcp
  end

  if getGoogleMetaData("project/project-id")
    @@is_in_gcp = true
    return true
  end
  @@is_in_gcp = false
  false
end
hosted_config() click to toggle source

If we're running this cloud, return the $MU_CFG blob we'd use to describe this environment as our target one.

# File modules/mu/providers/google.rb, line 107
def self.hosted_config
  return nil if !hosted?
  getGoogleMetaData("instance/zone").match(/^projects\/[^\/]+\/zones\/([^\/]+)$/)
  zone = Regexp.last_match[1]
  {
    "project" => MU::Cloud::Google.getGoogleMetaData("project/project-id"),
    "region" => zone.sub(/-[a-z]$/, "")
  }
end
iam(subclass = nil, credentials: nil) click to toggle source

Google's IAM Service API @param subclass [<Google::Apis::IamV1>]: If specified, will return the class ::Google::Apis::IamV1::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 846
def self.iam(subclass = nil, credentials: nil)
  require 'google/apis/iam_v1'

  if subclass.nil?
    @@iam_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "IamV1::IamService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials)
    return @@iam_api[credentials]
  elsif subclass.is_a?(Symbol)
    return Object.const_get("::Google").const_get("Apis").const_get("IamV1").const_get(subclass)
  end
end
initDeploy(deploy) click to toggle source

Do cloud-specific deploy instantiation tasks, such as copying SSH keys around, sticking secrets in buckets, creating resource groups, etc @param deploy [MU::MommaCat]

# File modules/mu/providers/google.rb, line 338
def self.initDeploy(deploy)
end
listAZs(region = self.myRegion) click to toggle source

List the Availability Zones associated with a given Google Cloud region. If no region is given, search the one in which this MU master server resides (if it resides in this cloud provider's ecosystem). @param region [String]: The region to search. @return [Array<String>]: The Availability Zones in this region.

# File modules/mu/providers/google.rb, line 808
def self.listAZs(region = self.myRegion)
  return [] if !credConfig
  MU::Cloud::Google.listRegions if !@@regions.has_key?(region)
  if !@@regions.has_key?(region)
    MU.log "Failed to get GCP region #{region}", MU::ERR, details: @@regions
    raise MuError, "No such Google Cloud region '#{region}'" if !@@regions.has_key?(region)
  end
  @@regions[region]
end
listCredentials() click to toggle source

Return the name strings of all known sets of credentials for this cloud @return [Array<String>]

# File modules/mu/providers/google.rb, line 142
def self.listCredentials
  if !$MU_CFG['google']
    return hosted? ? ["#default"] : nil
  end

  $MU_CFG['google'].keys
end
listHabitats(credentials = nil, use_cache: true) click to toggle source

List all Google Cloud Platform projects available to our credentials

# File modules/mu/providers/google.rb, line 713
def self.listHabitats(credentials = nil, use_cache: true)
  cfg = credConfig(credentials)
  return [] if !cfg
  if cfg['restrict_to_habitats'] and cfg['restrict_to_habitats'].is_a?(Array)
    cfg['restrict_to_habitats'] << cfg['project'] if cfg['project']
    return cfg['restrict_to_habitats'].uniq
  end
  if @allprojects and !@allprojects.empty? and use_cache
    return @allprojects
  end
  result = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects
  result.projects.reject! { |p| p.lifecycle_state == "DELETE_REQUESTED" }
  @allprojects = result.projects.map { |p| p.project_id }
  if cfg['ignore_habitats'] and cfg['ignore_habitats'].is_a?(Array)
    @allprojects.reject! { |p| cfg['ignore_habitats'].include?(p) }
  end
  @allprojects
end
listInstanceTypes(region = self.myRegion, credentials: nil, project: MU::Cloud::Google.defaultProject) click to toggle source

Query the GCP API for the list of valid Compute instance types and some of their attributes. We can use this in config validation and to help “translate” machine types across cloud providers. @param region [String]: Supported machine types can vary from region to region, so we look for the set we're interested in specifically @return [Hash]

# File modules/mu/providers/google.rb, line 770
def self.listInstanceTypes(region = self.myRegion, credentials: nil, project: MU::Cloud::Google.defaultProject)
  return {} if !credConfig(credentials)
  if @@instance_types and
     @@instance_types[project] and
     @@instance_types[project][region]
    return @@instance_types
  end

  return {} if !project

  @@instance_types ||= {}
  @@instance_types[project] ||= {}
  @@instance_types[project][region] ||= {}
  result = MU::Cloud::Google.compute(credentials: credentials).list_machine_types(project, listAZs(region).first)
  result.items.each { |type|
    @@instance_types[project][region][type.name] ||= {}
    @@instance_types[project][region][type.name]["memory"] = sprintf("%.1f", type.memory_mb/1024.0).to_f
    @@instance_types[project][region][type.name]["vcpu"] = type.guest_cpus.to_f
    if type.is_shared_cpu
      @@instance_types[project][region][type.name]["ecu"] = "Variable"
    else
      @@instance_types[project][region][type.name]["ecu"] = type.guest_cpus
    end
  }
  @@instance_types
end
listRegions(us_only = false, credentials: nil) click to toggle source

List all known Google Cloud Platform regions @param us_only [Boolean]: Restrict results to United States only

# File modules/mu/providers/google.rb, line 735
def self.listRegions(us_only = false, credentials: nil)
  if !MU::Cloud::Google.defaultProject(credentials)
    return []
  end
  if @@regions.size == 0
    begin
      result = MU::Cloud::Google.compute(credentials: credentials).list_regions(MU::Cloud::Google.defaultProject(credentials))
    rescue ::Google::Apis::ClientError => e
      if e.message.match(/forbidden/)
        raise MuError, "Insufficient permissions to list Google Cloud region. The service account #{myServiceAccount} should probably have the project owner role."
      end
      raise e
    end

    result.items.each { |region|
      @@regions[region.name] = []
      region.zones.each { |az|
        @@regions[region.name] << az.sub(/^.*?\/([^\/]+)$/, '\1')
      }
    }
  end
  if us_only
    @@regions.keys.delete_if { |r| !r.match(/^us/) }
  else
    @@regions.keys
  end
end
loadCredentials(scopes = nil, credentials: nil) click to toggle source

Pull our global Google Cloud Platform credentials out of their secure vault, feed them to the googleauth gem, and stash the results on hand for consumption by the various GCP APIs. @param scopes [Array<String>]: One or more scopes for which to authorizer the caller. Will vary depending on the API you're calling.

# File modules/mu/providers/google.rb, line 534
def self.loadCredentials(scopes = nil, credentials: nil)
  if @@authorizers[credentials] and @@authorizers[credentials][scopes.to_s]
    return @@authorizers[credentials][scopes.to_s]
  end

  cfg = credConfig(credentials)

  if cfg
    if cfg['project']
      @@enable_semaphores[cfg['project']] ||= Mutex.new
    end
    data = nil
    @@authorizers[credentials] ||= {}
  
    def self.get_machine_credentials(scopes, credentials = nil)
      @@svc_account_name = MU::Cloud::Google.getGoogleMetaData("instance/service-accounts/default/email")
      MU.log "We are hosted in GCP, so I will attempt to use the service account #{@@svc_account_name} to make API requests.", MU::DEBUG

      @@authorizers[credentials][scopes.to_s] = ::Google::Auth.get_application_default(scopes)
      @@authorizers[credentials][scopes.to_s].fetch_access_token!
      @@default_project ||= MU::Cloud::Google.getGoogleMetaData("project/project-id")
      begin
        listRegions(credentials: credentials)
        listInstanceTypes(credentials: credentials)
        listHabitats(credentials)
      rescue ::Google::Apis::ClientError
        MU.log "Found machine credentials #{@@svc_account_name}, but these don't appear to have sufficient permissions or scopes", MU::WARN, details: scopes
        @@authorizers.delete(credentials)
        return nil
      end
      @@authorizers[credentials][scopes.to_s]
    end

    if cfg["credentials_file"] or cfg["credentials_encoded"]

      begin
        data = if cfg["credentials_encoded"]
          JSON.parse(Base64.decode64(cfg["credentials_encoded"]))
        else
          JSON.parse(File.read(cfg["credentials_file"]))
        end
        @@default_project ||= data["project_id"]
        creds = {
          :json_key_io => StringIO.new(MultiJson.dump(data)),
          :scope => scopes
        }
        @@svc_account_name = data["client_email"]
        @@authorizers[credentials][scopes.to_s] = ::Google::Auth::ServiceAccountCredentials.make_creds(creds)
        return @@authorizers[credentials][scopes.to_s]
      rescue JSON::ParserError, Errno::ENOENT, Errno::EACCES => e
        if !MU::Cloud::Google.hosted?
          raise MuError, "Google Cloud credentials file #{cfg["credentials_file"]} is missing or invalid (#{e.message})"
        end
        MU.log "Google Cloud credentials file #{cfg["credentials_file"]} is missing or invalid", MU::WARN, details: e.message
        return get_machine_credentials(scopes, credentials)
      end
    elsif cfg["credentials"]
      begin
        vault, item = cfg["credentials"].split(/:/)
        data = MU::Groomer::Chef.getSecret(vault: vault, item: item).to_h
      rescue MU::Groomer::MuNoSuchSecret
        if !MU::Cloud::Google.hosted?
          raise MuError, "Google Cloud credentials not found in Vault #{vault}:#{item}"
        end
        MU.log "Google Cloud credentials not found in Vault #{vault}:#{item}", MU::WARN
        found = get_machine_credentials(scopes, credentials)
        raise MuError, "No valid credentials available! Either grant admin privileges to machine service account, or manually add a different one with mu-configure" if found.nil?
        return found
      end

      @@default_project ||= data["project_id"]
      creds = {
        :json_key_io => StringIO.new(MultiJson.dump(data)),
        :scope => scopes
      }
      @@svc_account_name = data["client_email"]
      @@authorizers[credentials][scopes.to_s] = ::Google::Auth::ServiceAccountCredentials.make_creds(creds)
      return @@authorizers[credentials][scopes.to_s]
    elsif MU::Cloud::Google.hosted?
      found = get_machine_credentials(scopes, credentials)
      raise MuError, "No valid credentials available! Either grant admin privileges to machine service account, or manually add a different one with mu-configure" if found.nil?
      return found
    else
      raise MuError, "Google Cloud credentials not configured"
    end

  end
  nil
end
logging(subclass = nil, credentials: nil) click to toggle source

Google's StackDriver Logging Service API @param subclass [<Google::Apis::LoggingV2>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 973
def self.logging(subclass = nil, credentials: nil)
  require 'google/apis/logging_v2'

  if subclass.nil?
    @@logging_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "LoggingV2::LoggingService", scopes: ['cloud-platform'], credentials: credentials)
    return @@logging_api[credentials]
  elsif subclass.is_a?(Symbol)
    return Object.const_get("::Google").const_get("Apis").const_get("LoggingV2").const_get(subclass)
  end
end
myProject() click to toggle source

If this Mu master resides in the Google Cloud Platform, return the project id in which we reside. Nil if we're not in GCP.

# File modules/mu/providers/google.rb, line 642
def self.myProject
  if MU::Cloud::Google.hosted?
    return MU::Cloud::Google.getGoogleMetaData("project/project-id")
  end
  nil
end
myRegion(credentials = nil) click to toggle source

If we've configured Google as a provider, or are simply hosted in GCP, decide what our default region is.

# File modules/mu/providers/google.rb, line 322
def self.myRegion(credentials = nil)
  cfg = credConfig(credentials)
  if cfg and cfg['region']
    @@myRegion_var = cfg['region']
  elsif MU::Cloud::Google.hosted?
    zone = MU::Cloud::Google.getGoogleMetaData("instance/zone")
    @@myRegion_var = zone.gsub(/^.*?\/|\-\d+$/, "")
  else
    @@myRegion_var = "us-east4"
  end
  @@myRegion_var
end
myServiceAccount() click to toggle source

If this Mu master resides in the Google Cloud Platform, return the default service account associated with its metadata.

# File modules/mu/providers/google.rb, line 651
def self.myServiceAccount
  if MU::Cloud::Google.hosted?
    MU::Cloud::Google.getGoogleMetaData("instance/service-accounts/default/email")
  else
    nil
  end
end
myVPCObj() click to toggle source

If we reside in this cloud, return the VPC in which we, the Mu Master, reside. @return [MU::Cloud::VPC]

# File modules/mu/providers/google.rb, line 131
def self.myVPCObj
  return nil if !hosted?
  instance = MU.myCloudDescriptor
  return nil if !instance or !instance.network_interfaces or instance.network_interfaces.size == 0
  vpc = MU::MommaCat.findStray("Google", "vpc", cloud_id: instance.network_interfaces.first.network.gsub(/.*?\/([^\/]+)$/, '\1'), dummy_ok: true, habitats: [myProject])
  return nil if vpc.nil? or vpc.size == 0
  vpc.first
end
nameStr(name) click to toggle source

Google has fairly strict naming conventions (all lowercase, no underscores, etc). Provide a wrapper to our standard names to handle it.

# File modules/mu/providers/google.rb, line 799
def self.nameStr(name)
  name.downcase.gsub(/[^a-z0-9\-]/, "-")
end
projectLookup(name, deploy = MU.mommacat, raise_on_fail: true, sibling_only: false) click to toggle source

A shortcut for {MU::MommaCat.findStray} to resolve a shorthand project name into a cloud object, whether it refers to a sibling by internal name or by cloud identifier. @param name [String] @param deploy [String] @param raise_on_fail [Boolean] @param sibling_only [Boolean] @return [MU::Config::Habitat,nil]

# File modules/mu/providers/google.rb, line 238
def self.projectLookup(name, deploy = MU.mommacat, raise_on_fail: true, sibling_only: false)
  project_obj = deploy.findLitterMate(type: "habitats", name: name) if deploy and caller.grep(/`findLitterMate'/).empty? # XXX the dumbest

  if !project_obj and !sibling_only
    resp = MU::MommaCat.findStray(
      "Google",
      "habitats",
      deploy_id: deploy ? deploy.deploy_id : nil,
      cloud_id: name,
      name: name,
      dummy_ok: true
    )

    project_obj = resp.first if resp and resp.size > 0
  end

  if (!project_obj or !project_obj.cloud_id) and raise_on_fail
    raise MuError, "Failed to find project '#{name}' in deploy #{deploy.deploy_id}"
  end

  project_obj
end
projectToRef(project, config: nil, credentials: nil) click to toggle source

Take a plain string that might be a reference to sibling project declared elsewhere in the active stack, or the project id of a live cloud resource, and return a {MU::Config::Ref} object @param project [String]: The name of a sibling project, or project id of an active project in GCP @param config [MU::Config]: A {MU::Config} object containing sibling resources, typically what we'd pass if we're calling during configuration parsing @param credentials [String]: @return [MU::Config::Ref]

# File modules/mu/providers/google.rb, line 196
def self.projectToRef(project, config: nil, credentials: nil)
  return nil if !project

  if config and config.haveLitterMate?(project, "habitat")
    ref = MU::Config::Ref.new(
      name: project,
      cloud: "Google",
      credentials: credentials,
      type: "habitats"
    )
  end
  
  if !ref
    resp = MU::MommaCat.findStray(
      "Google",
      "habitats",
      cloud_id: project,
      credentials: credentials,
      dummy_ok: true
    )
    if resp and resp.size > 0
      project_obj = resp.first
      ref = MU::Config::Ref.new(
        id: project_obj.cloud_id,
        cloud: "Google",
        credentials: credentials,
        type: "habitats"
      )
    end
  end

  ref
end
removeDeploySecretsAndRoles(deploy_id = MU.deploy_id, flags: {}, noop: false, credentials: nil) click to toggle source

Remove the service account and various deploy secrets associated with a deployment. Intended for invocation from MU::Cleanup. @param deploy_id [String]: The deploy for which we're granting the secret @param noop [Boolean]: If true, will only print what would be done

# File modules/mu/providers/google.rb, line 378
def self.removeDeploySecretsAndRoles(deploy_id = MU.deploy_id, flags: {}, noop: false, credentials: nil)
  cfg = credConfig(credentials)
  return if !cfg or !cfg['project']
  flags["project"] ||= cfg['project']

  resp = MU::Cloud::Google.storage(credentials: credentials).list_objects(
    adminBucketName(credentials),
    prefix: deploy_id
  )
  if resp and resp.items
    resp.items.each { |obj|
      MU.log "Deleting gs://#{adminBucketName(credentials)}/#{obj.name}"
      if !noop
        MU::Cloud::Google.storage(credentials: credentials).delete_object(
          adminBucketName(credentials),
          obj.name
        )
      end
    }
  end
end
required_instance_methods() click to toggle source

Any cloud-specific instance methods we require our resource implementations to have, above and beyond the ones specified by {MU::Cloud} @return [Array<Symbol>]

# File modules/mu/providers/google.rb, line 51
def self.required_instance_methods
  [:url]
end
resourceInitHook(cloudobj, deploy) click to toggle source

A hook that is always called just before any of the instance method of our resource implementations gets invoked, so that we can ensure that repetitive setup tasks (like resolving :resource_group for Azure resources) have always been done. @param cloudobj [MU::Cloud] @param deploy [MU::MommaCat]

# File modules/mu/providers/google.rb, line 77
      def self.resourceInitHook(cloudobj, deploy)
        class << self
          attr_reader :project_id
          attr_reader :customer
          # url is too complex for an attribute (we get it from the cloud API),
          # so it's up in AdditionalResourceMethods instead
        end
        return if !cloudobj

        cloudobj.instance_variable_set(:@customer, MU::Cloud::Google.customerID(cloudobj.config['credentials']))

# XXX ensure @cloud_id and @project_id if this is a habitat
# XXX skip project_id if this is a folder or group
        if deploy
# XXX this may be wrong for new deploys (but def right for regrooms)
          project = MU::Cloud::Google.projectLookup(cloudobj.config['project'], deploy, sibling_only: true, raise_on_fail: false)
          project_id = project.nil? ? cloudobj.config['project'] : project.cloudobj.cloud_id
          cloudobj.instance_variable_set(:@project_id, project_id)
        else
          cloudobj.instance_variable_set(:@project_id, cloudobj.config['project'])
        end

# XXX @url? Well we're not likely to have @cloud_desc at this point, so maybe
# that needs to be a generic-to-google wrapper like def url; cloud_desc.self_link;end

# XXX something like: vpc["habitat"] = MU::Cloud::Google.projectToRef(vpc["project"], config: configurator, credentials: vpc["credentials"])
      end
resource_manager(subclass = nil, credentials: nil) click to toggle source

Google's Cloud Resource Manager API @param subclass [<Google::Apis::CloudresourcemanagerV1>]: If specified, will return the class ::Google::Apis::CloudresourcemanagerV1::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 892
def self.resource_manager(subclass = nil, credentials: nil)
  require 'google/apis/cloudresourcemanager_v1'

  if subclass.nil?
    if !MU::Cloud::Google.credConfig(credentials)
      raise MuError, "No such credential set #{credentials} defined in mu.yaml!"
    end
    @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
    return @@resource_api[credentials]
  elsif subclass.is_a?(Symbol)
    return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV1").const_get(subclass)
  end
end
service_manager(subclass = nil, credentials: nil) click to toggle source

Google's Service Manager API (the one you use to enable pre-project APIs) @param subclass [<Google::Apis::ServicemanagementV1>]: If specified, will return the class ::Google::Apis::ServicemanagementV1::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 934
def self.service_manager(subclass = nil, credentials: nil)
  require 'google/apis/servicemanagement_v1'

  if subclass.nil?
    @@service_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "ServicemanagementV1::ServiceManagementService", scopes: ['cloud-platform'], credentials: credentials)
    return @@service_api[credentials]
  elsif subclass.is_a?(Symbol)
    return Object.const_get("::Google").const_get("Apis").const_get("ServicemanagementV1").const_get(subclass)
  end
end
sql(subclass = nil, credentials: nil) click to toggle source

Google's SQL Service API @param subclass [<Google::Apis::SqladminV1beta4>]: If specified, will return the class ::Google::Apis::SqladminV1beta4::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 947
def self.sql(subclass = nil, credentials: nil)
  require 'google/apis/sqladmin_v1beta4'

  if subclass.nil?
    @@sql_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "SqladminV1beta4::SQLAdminService", scopes: ['cloud-platform'], credentials: credentials)
    return @@sql_api[credentials]
  elsif subclass.is_a?(Symbol)
    return Object.const_get("::Google").const_get("Apis").const_get("SqladminV1beta4").const_get(subclass)
  end
end
storage(subclass = nil, credentials: nil) click to toggle source

Google's Storage Service API @param subclass [<Google::Apis::StorageV1>]: If specified, will return the class ::Google::Apis::StorageV1::subclass instead of an API client instance

# File modules/mu/providers/google.rb, line 833
def self.storage(subclass = nil, credentials: nil)
  require 'google/apis/storage_v1'

  if subclass.nil?
    @@storage_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "StorageV1::StorageService", scopes: ['cloud-platform'], credentials: credentials)
    return @@storage_api[credentials]
  elsif subclass.is_a?(Symbol)
    return Object.const_get("::Google").const_get("Apis").const_get("StorageV1").const_get(subclass)
  end
end
svc_account_name() click to toggle source

Fetch the name of the service account we were using last time we loaded GCP credentials. @return [String]

# File modules/mu/providers/google.rb, line 527
def self.svc_account_name
  @@svc_account_name
end
virtual?() click to toggle source

Is this a “real” cloud provider, or a stub like CloudFormation?

# File modules/mu/providers/google.rb, line 56
def self.virtual?
  false
end
writeDeploySecret(deploy, value, name = nil, credentials: nil) click to toggle source

Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it @param deploy_id [String]: The deploy for which we're writing the secret @param value [String]: The contents of the secret

# File modules/mu/providers/google.rb, line 351
def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
  deploy_id = deploy.deploy_id
  name ||= deploy_id+"-secret"
  begin
    MU.log "Writing #{name} to Cloud Storage bucket #{adminBucketName(credentials)}"

    f = Tempfile.new(name) # XXX this is insecure and stupid
    f.write value
    f.close
    objectobj = MU::Cloud::Google.storage(:Object).new(
      bucket: adminBucketName(credentials),
      name: name
    )
    MU::Cloud::Google.storage(credentials: credentials).insert_object(
      adminBucketName(credentials),
      objectobj,
      upload_source: f.path
    )
    f.unlink
  rescue ::Google::Apis::ClientError => e
    raise MU::MommaCat::DeployInitializeError, "Got #{e.inspect} trying to write #{name} to #{adminBucketName(credentials)}"
  end
end