class MU::Cloud::Google
Attributes
Public Class Methods
Resolve the administrative Cloud
Storage bucket for a given credential set, or return a default. @param credentials [String] @return [String]
# File modules/mu/providers/google.rb, line 265 def self.adminBucketName(credentials = nil) #XXX find a default if this particular account doesn't have a log_bucket_name configured cfg = credConfig(credentials) if cfg.nil? raise MuError, "Failed to load Google credential set #{credentials}" end cfg['log_bucket_name'] end
Resolve the administrative Cloud
Storage bucket for a given credential set, or return a default. @param credentials [String] @return [String]
# File modules/mu/providers/google.rb, line 278 def self.adminBucketUrl(credentials = nil) "gs://"+adminBucketName(credentials)+"/" end
GCP's AdminDirectory Service API @param subclass [<Google::Apis::AdminDirectoryV1>]: If specified, will return the class ::Google::Apis::AdminDirectoryV1::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 859 def self.admin_directory(subclass = nil, credentials: nil) require 'google/apis/admin_directory_v1' # fill in the default credential set name so we don't generate # dopey extra warnings about falling back on scopes credentials ||= MU::Cloud::Google.credConfig(credentials, name_only: true) writescopes = ['admin.directory.group.member', 'admin.directory.group', 'admin.directory.user', 'admin.directory.domain', 'admin.directory.orgunit', 'admin.directory.rolemanagement', 'admin.directory.customer', 'admin.directory.user.alias', 'admin.directory.userschema'] readscopes = ['admin.directory.group.member.readonly', 'admin.directory.group.readonly', 'admin.directory.user.readonly', 'admin.directory.domain.readonly', 'admin.directory.orgunit.readonly', 'admin.directory.rolemanagement.readonly', 'admin.directory.customer.readonly', 'admin.directory.user.alias.readonly', 'admin.directory.userschema.readonly'] @@readonly_semaphore.synchronize { use_scopes = readscopes+writescopes if @@readonly[credentials] and @@readonly[credentials]["AdminDirectoryV1"] use_scopes = readscopes.dup end if subclass.nil? begin @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: use_scopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials, auth_error_quiet: true) rescue Signet::AuthorizationError MU.log "Falling back to read-only access to DirectoryService API for credential set '#{credentials}'", MU::WARN @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: readscopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials) @@readonly[credentials] ||= {} @@readonly[credentials]["AdminDirectoryV1"] = true end return @@admin_directory_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("AdminDirectoryV1").const_get(subclass) end } end
Google's Cloud
Billing Budget Service API @param subclass [<Google::Apis::CloudbillingV1>]: If specified, will return the class ::Google::Apis::CloudbillingV1::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 999 def self.billing(subclass = nil, credentials: nil) require 'google/apis/cloudbilling_v1' if subclass.nil? @@billing_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudbillingV1::CloudbillingService", scopes: ['cloud-platform', 'cloud-billing'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as']) return @@billing_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("CloudbillingV1").const_get(subclass) end end
Google's Cloud
Billing Service API @param subclass [<Google::Apis::CloudbillingV1>]: If specified, will return the class ::Google::Apis::CloudbillingV1::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 986 def self.budgets(subclass = nil, credentials: nil) require 'google/apis/billingbudgets_v1' if subclass.nil? @@budgets_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "BillingbudgetsV1::CloudBillingBudgetService", scopes: ['cloud-platform', 'cloud-billing'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as']) return @@budgets_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("BillingbudgetsV1").const_get(subclass) end end
Purge cloud-specific deploy meta-artifacts (SSH keys, resource groups, etc) @param deploy_id [MU::MommaCat]
# File modules/mu/providers/google.rb, line 344 def self.cleanDeploy(deploy_id, credentials: nil, noop: false) removeDeploySecretsAndRoles(deploy_id, noop: noop, credentials: credentials) end
Google's Compute Service API @param subclass [<Google::Apis::ComputeV1>]: If specified, will return the class ::Google::Apis::ComputeV1::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 820 def self.compute(subclass = nil, credentials: nil) require 'google/apis/compute_v1' if subclass.nil? @@compute_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "ComputeV1::ComputeService", scopes: ['cloud-platform', 'compute.readonly'], credentials: credentials) return @@compute_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("ComputeV1").const_get(subclass) end end
A non-working example configuration
# File modules/mu/providers/google.rb, line 118 def self.config_example sample = hosted_config sample ||= { "project" => "my-project", "region" => "us-east4" } sample["credentials_file"] = "#{Etc.getpwuid(Process.uid).dir}/gcp_serviceacct.json" sample["log_bucket_name"] = "my-mu-cloud-storage-bucket" sample end
Google's Container API @param subclass [<Google::Apis::ContainerV1>]: If specified, will return the class ::Google::Apis::ContainerV1::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 921 def self.container(subclass = nil, credentials: nil) require 'google/apis/container_v1' if subclass.nil? @@container_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "ContainerV1::ContainerService", scopes: ['cloud-platform'], credentials: credentials) return @@container_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("ContainerV1").const_get(subclass) end end
Create an SSL Certificate resource from some local x509 cert files. @param name [String]: A resource name for the certificate @param cert [String,OpenSSL::X509::Certificate]: An x509 certificate @param key [String,OpenSSL::PKey]: An x509 private key @return [Google::Apis::ComputeV1::SslCertificate]
# File modules/mu/providers/google.rb, line 511 def self.createSSLCertificate(name, cert, key, flags = {}, credentials: nil) flags["project"] ||= MU::Cloud::Google.defaultProject(credentials) flags["description"] ||= MU.deploy_id certobj = ::Google::Apis::ComputeV1::SslCertificate.new( name: name, certificate: cert.to_s, private_key: key.to_s, description: flags["description"] ) MU::Cloud::Google.compute(credentials: credentials).insert_ssl_certificate(flags["project"], certobj) end
Return the $MU_CFG data associated with a particular profile/name/set of credentials. If no account name is specified, will return one flagged as default. Returns nil if GCP is not configured. Throws an exception if an account name is specified which does not exist. @param name [String]: The name of the key under 'google' in mu.yaml to return @return [Hash,nil]
# File modules/mu/providers/google.rb, line 288 def self.credConfig(name = nil, name_only: false) # If there's nothing in mu.yaml (which is wrong), but we're running # on a machine hosted in GCP, fake it with that machine's service # account and hope for the best. if !$MU_CFG['google'] or !$MU_CFG['google'].is_a?(Hash) or $MU_CFG['google'].size == 0 return @@my_hosted_cfg if @@my_hosted_cfg if hosted? @@my_hosted_cfg = hosted_config return name_only ? "#default" : @@my_hosted_cfg end return nil end if name.nil? $MU_CFG['google'].each_pair { |set, cfg| if cfg['default'] return name_only ? set : cfg end } else if $MU_CFG['google'][name] return name_only ? name : $MU_CFG['google'][name] elsif @@acct_to_profile_map[name.to_s] return name_only ? name : @@acct_to_profile_map[name.to_s] end # XXX whatever process might lead us to populate @@acct_to_profile_map with some mappings, like projectname -> account profile, goes here return nil end end
Fetch the GSuite/Cloud Identity customer id for the domain associated with the given credentials, if a domain is set via the masquerade_as
configuration option.
# File modules/mu/providers/google.rb, line 1082 def self.customerID(credentials = nil) cfg = credConfig(credentials) if !cfg or !cfg['masquerade_as'] return nil end if @@customer_ids_cache[credentials] return @@customer_ids_cache[credentials] end user = MU::Cloud::Google.admin_directory(credentials: credentials).get_user(cfg['masquerade_as']) if user and user.customer_id @@customer_ids_cache[credentials] = user.customer_id end @@customer_ids_cache[credentials] end
We want a default place to put new projects for the Habitat
resource, so if we have a root folder, we can go ahead and use that. @param credentials [String] @return [String]
# File modules/mu/providers/google.rb, line 699 def self.defaultFolder(credentials = nil) project = defaultProject(credentials) resp = MU::Cloud::Google.resource_manager(credentials: credentials).get_project_ancestry(project) resp.ancestor.each { |a| if a.resource_id.type == "folder" return a.resource_id.id end } nil end
Our credentials map to a project, an organizational structure in Google
Cloud
. This fetches the identifier of the project associated with our default credentials. @param credentials [String] @return [String]
# File modules/mu/providers/google.rb, line 666 def self.defaultProject(credentials = nil) if @@default_project_cache.has_key?(credentials) return @@default_project_cache[credentials] end cfg = credConfig(credentials) if !cfg or !cfg['project'] if hosted? @@default_project_cache[credentials] = myProject return myProject end if cfg begin result = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects result.projects.reject! { |p| p.lifecycle_state == "DELETE_REQUESTED" } available = result.projects.map { |p| p.project_id } if available.size == 1 @@default_project_cache[credentials] = available[0] return available[0] end rescue # fine end end end return nil if !cfg loadCredentials(credentials) if !@@authorizers[credentials] @@default_project_cache[credentials] = cfg['project'] cfg['project'] end
Most of our resource implementation find
methods have to mangle their args to make sure they've extracted a project or location argument from other available information. This does it for them. @return [Hash]
# File modules/mu/providers/google.rb, line 64 def self.findLocationArgs(**args) args[:project] ||= args[:habitat] args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials]) args[:location] ||= args[:region] || args[:availability_zone] || "-" args end
Google's Firestore (NoSQL) Service API @param subclass [<Google::Apis::FirestoreV1>]: If specified, will return the class ::Google::Apis::FirestoreV1::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 960 def self.firestore(subclass = nil, credentials: nil) require 'google/apis/firestore_v1' if subclass.nil? @@firestore_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "FirestoreV1::FirestoreService", scopes: ['cloud-platform'], credentials: credentials) return @@firestore_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("FirestoreV1").const_get(subclass) end end
Google's Cloud
Resource Manager API V2, which apparently has all the folder bits @param subclass [<Google::Apis::CloudresourcemanagerV2>]: If specified, will return the class ::Google::Apis::CloudresourcemanagerV2::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 908 def self.folder(subclass = nil, credentials: nil) require 'google/apis/cloudresourcemanager_v2' if subclass.nil? @@resource2_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV2::CloudResourceManagerService", scopes: ['cloud-platform', 'cloudplatformfolders'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as']) return @@resource2_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV2").const_get(subclass) end end
Google's Cloud
Function
Service API @param subclass [<Google::Apis::CloudfunctionsV1>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 1012 def self.function(subclass = nil, credentials: nil) require 'google/apis/cloudfunctions_v1' if subclass.nil? @@function_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudfunctionsV1::CloudFunctionsService", scopes: ['cloud-platform'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as']) return @@function_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("CloudfunctionsV1").const_get(subclass) end end
Fetch a URL
# File modules/mu/providers/google.rb, line 625 def self.get(url) uri = URI url resp = nil Net::HTTP.start(uri.host, uri.port) do |http| resp = http.get(uri) end unless resp.code == "200" puts resp.code, resp.body exit end resp.body end
Retrieve the domains, if any, which these credentials can manage via GSuite or Cloud
Identity. @param credentials [String] @return [Array<String>],nil]
# File modules/mu/providers/google.rb, line 1027 def self.getDomains(credentials = nil) my_org = getOrg(credentials) return nil if !my_org resp = MU::Cloud::Google.admin_directory(credentials: credentials).list_domains(MU::Cloud::Google.customerID(credentials)) resp.domains.map { |d| d.domain_name.downcase } end
Fetch a Google
instance metadata parameter (example: instance/id). @param param [String]: The parameter name to fetch @return [String, nil]
# File modules/mu/providers/google.rb, line 487 def self.getGoogleMetaData(param) base_url = "http://metadata.google.internal/computeMetadata/v1" begin Timeout.timeout(2) do response = URI.open( "#{base_url}/#{param}", "Metadata-Flavor" => "Google" ).read return response end rescue Net::HTTPServerException, OpenURI::HTTPError, Timeout::Error, SocketError, Errno::EHOSTUNREACH, Errno::ENETUNREACH => e # This is fairly normal, just handle it gracefully logger = MU::Logger.new logger.log "Failed metadata request #{base_url}/#{param}: #{e.inspect}", MU::DEBUG end nil end
Retrieve the organization, if any, to which these credentials belong. @param credentials [String] @return [Array<OpenStruct>],nil]
# File modules/mu/providers/google.rb, line 1039 def self.getOrg(credentials = nil, with_id: nil) creds = MU::Cloud::Google.credConfig(credentials) return nil if !creds credname = if creds and creds['name'] creds['name'] else "default" end with_id ||= creds['org'] if creds['org'] return @@orgmap[credname] if @@orgmap.has_key?(credname) resp = MU::Cloud::Google.resource_manager(credentials: credname).search_organizations if resp and resp.organizations # XXX no idea if it's possible to be a member of multiple orgs if !with_id @@orgmap[credname] = resp.organizations.first return resp.organizations.first else resp.organizations.each { |org| if org.name == with_id or org.display_name == with_id or org.name == "organizations/#{with_id}" @@orgmap[credname] = org return org end } return nil end end @@orgmap[credname] = nil MU.log "Unable to list_organizations with credentials #{credname}. If this account is part of a GSuite or Cloud Identity domain, verify that Oauth delegation is properly configured and that 'masquerade_as' is properly set for the #{credname} Google credential set in mu.yaml.", MU::ERR, details: ["https://cloud.google.com/resource-manager/docs/creating-managing-organization", "https://admin.google.com/AdminHome?chromeless=1#OGX:ManageOauthClients"] nil end
# File modules/mu/providers/google.rb, line 548 def self.get_machine_credentials(scopes, credentials = nil) @@svc_account_name = MU::Cloud::Google.getGoogleMetaData("instance/service-accounts/default/email") MU.log "We are hosted in GCP, so I will attempt to use the service account #{@@svc_account_name} to make API requests.", MU::DEBUG @@authorizers[credentials][scopes.to_s] = ::Google::Auth.get_application_default(scopes) @@authorizers[credentials][scopes.to_s].fetch_access_token! @@default_project ||= MU::Cloud::Google.getGoogleMetaData("project/project-id") begin listRegions(credentials: credentials) listInstanceTypes(credentials: credentials) listHabitats(credentials) rescue ::Google::Apis::ClientError MU.log "Found machine credentials #{@@svc_account_name}, but these don't appear to have sufficient permissions or scopes", MU::WARN, details: scopes @@authorizers.delete(credentials) return nil end @@authorizers[credentials][scopes.to_s] end
Grant access to appropriate Cloud
Storage objects in our log/secret bucket for a deploy member. @param acct [String]: The service account (by email addr) to which we'll grant access @param deploy_id [String]: The deploy for which we're granting the secret XXX add equivalent for AWS
and call agnostically
# File modules/mu/providers/google.rb, line 404 def self.grantDeploySecretAccess(acct, deploy_id = MU.deploy_id, name = nil, credentials: nil) name ||= deploy_id+"-secret" aclobj = nil retries = 0 begin MU.log "Granting #{acct} access to list Cloud Storage bucket #{adminBucketName(credentials)}" MU::Cloud::Google.storage(credentials: credentials).insert_bucket_access_control( adminBucketName(credentials), MU::Cloud::Google.storage(:BucketAccessControl).new( bucket: adminBucketName(credentials), role: "READER", entity: "user-"+acct ) ) aclobj = MU::Cloud::Google.storage(:ObjectAccessControl).new( bucket: adminBucketName(credentials), role: "READER", entity: "user-"+acct ) [name].each { |obj| MU.log "Granting #{acct} access to #{obj} in Cloud Storage bucket #{adminBucketName(credentials)}" MU::Cloud::Google.storage(credentials: credentials).insert_object_access_control( adminBucketName(credentials), obj, aclobj ) } rescue ::Google::Apis::ClientError => e MU.log e.message, MU::WARN, details: e.inspect if e.inspect.match(/body: "Not Found"/) raise MuError, "Google admin bucket #{adminBucketName(credentials)} or key #{name} does not appear to exist or is not visible with #{credentials ? credentials : "default"} credentials" elsif e.message.match(/notFound: |Unknown user:|conflict: /) if retries < 5 sleep 5 retries += 1 retry else raise e end elsif e.inspect.match(/The metadata for object "null" was edited during the operation/) MU.log e.message+" - Google admin bucket #{adminBucketName(credentials)}/#{name} with #{credentials ? credentials : "default"} credentials", MU::DEBUG, details: aclobj sleep 10 retry else raise MuError, "Got #{e.message} trying to set ACLs for #{deploy_id} in #{adminBucketName(credentials)}" end end end
Return what we think of as a cloud object's habitat. In GCP, this means the project_id
in which is resident. If this is not applicable, such as for a {Habitat} or {Folder}, returns nil. @param cloudobj [MU::Cloud::Google]: The resource from which to extract the habitat id @return [String,nil]
# File modules/mu/providers/google.rb, line 157 def self.habitat(cloudobj, nolookup: false, deploy: nil) @@habmap ||= {} # XXX whaddabout config['habitat'] HNNNGH return nil if !cloudobj.cloudclass.canLiveIn.include?(:Habitat) # XXX these are assholes because they're valid two different ways ugh ugh return nil if [MU::Cloud::Google::Group, MU::Cloud::Google::Folder].include?(cloudobj.cloudclass) if cloudobj.config and cloudobj.config['project'] if nolookup return cloudobj.config['project'] end if @@habmap[cloudobj.config['project']] return @@habmap[cloudobj.config['project']] end deploy ||= cloudobj.deploy if cloudobj.respond_to?(:deploy) projectobj = projectLookup(cloudobj.config['project'], deploy, raise_on_fail: false) if projectobj @@habmap[cloudobj.config['project']] = projectobj.cloud_id return projectobj.cloud_id end end # blow up if this resource *has* to live in a project if cloudobj.cloudclass.canLiveIn == [:Habitat] MU.log "Failed to find project for cloudobj #{cloudobj.to_s}", MU::ERR, details: cloudobj raise MuError, "Failed to find project for cloudobj #{cloudobj.to_s}" end nil end
Alias for #{MU::Cloud::Google.hosted?}
# File modules/mu/providers/google.rb, line 460 def self.hosted MU::Cloud::Google.hosted? end
Determine whether we (the Mu master, presumably) are hosted in this cloud. @return [Boolean]
# File modules/mu/providers/google.rb, line 467 def self.hosted? if $MU_CFG.has_key?("google_is_hosted") @@is_in_aws = $MU_CFG["google_is_hosted"] return $MU_CFG["google_is_hosted"] end if !@@is_in_gcp.nil? return @@is_in_gcp end if getGoogleMetaData("project/project-id") @@is_in_gcp = true return true end @@is_in_gcp = false false end
If we're running this cloud, return the $MU_CFG blob we'd use to describe this environment as our target one.
# File modules/mu/providers/google.rb, line 107 def self.hosted_config return nil if !hosted? getGoogleMetaData("instance/zone").match(/^projects\/[^\/]+\/zones\/([^\/]+)$/) zone = Regexp.last_match[1] { "project" => MU::Cloud::Google.getGoogleMetaData("project/project-id"), "region" => zone.sub(/-[a-z]$/, "") } end
Google's IAM Service API @param subclass [<Google::Apis::IamV1>]: If specified, will return the class ::Google::Apis::IamV1::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 846 def self.iam(subclass = nil, credentials: nil) require 'google/apis/iam_v1' if subclass.nil? @@iam_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "IamV1::IamService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials) return @@iam_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("IamV1").const_get(subclass) end end
Do cloud-specific deploy instantiation tasks, such as copying SSH keys around, sticking secrets in buckets, creating resource groups, etc @param deploy [MU::MommaCat]
# File modules/mu/providers/google.rb, line 338 def self.initDeploy(deploy) end
List the Availability Zones associated with a given Google
Cloud
region. If no region is given, search the one in which this MU
master server resides (if it resides in this cloud provider's ecosystem). @param region [String]: The region to search. @return [Array<String>]: The Availability Zones in this region.
# File modules/mu/providers/google.rb, line 808 def self.listAZs(region = self.myRegion) return [] if !credConfig MU::Cloud::Google.listRegions if !@@regions.has_key?(region) if !@@regions.has_key?(region) MU.log "Failed to get GCP region #{region}", MU::ERR, details: @@regions raise MuError, "No such Google Cloud region '#{region}'" if !@@regions.has_key?(region) end @@regions[region] end
Return the name strings of all known sets of credentials for this cloud @return [Array<String>]
# File modules/mu/providers/google.rb, line 142 def self.listCredentials if !$MU_CFG['google'] return hosted? ? ["#default"] : nil end $MU_CFG['google'].keys end
List all Google
Cloud
Platform projects available to our credentials
# File modules/mu/providers/google.rb, line 713 def self.listHabitats(credentials = nil, use_cache: true) cfg = credConfig(credentials) return [] if !cfg if cfg['restrict_to_habitats'] and cfg['restrict_to_habitats'].is_a?(Array) cfg['restrict_to_habitats'] << cfg['project'] if cfg['project'] return cfg['restrict_to_habitats'].uniq end if @allprojects and !@allprojects.empty? and use_cache return @allprojects end result = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects result.projects.reject! { |p| p.lifecycle_state == "DELETE_REQUESTED" } @allprojects = result.projects.map { |p| p.project_id } if cfg['ignore_habitats'] and cfg['ignore_habitats'].is_a?(Array) @allprojects.reject! { |p| cfg['ignore_habitats'].include?(p) } end @allprojects end
Query the GCP API for the list of valid Compute instance types and some of their attributes. We can use this in config validation and to help “translate” machine types across cloud providers. @param region [String]: Supported machine types can vary from region to region, so we look for the set we're interested in specifically @return [Hash]
# File modules/mu/providers/google.rb, line 770 def self.listInstanceTypes(region = self.myRegion, credentials: nil, project: MU::Cloud::Google.defaultProject) return {} if !credConfig(credentials) if @@instance_types and @@instance_types[project] and @@instance_types[project][region] return @@instance_types end return {} if !project @@instance_types ||= {} @@instance_types[project] ||= {} @@instance_types[project][region] ||= {} result = MU::Cloud::Google.compute(credentials: credentials).list_machine_types(project, listAZs(region).first) result.items.each { |type| @@instance_types[project][region][type.name] ||= {} @@instance_types[project][region][type.name]["memory"] = sprintf("%.1f", type.memory_mb/1024.0).to_f @@instance_types[project][region][type.name]["vcpu"] = type.guest_cpus.to_f if type.is_shared_cpu @@instance_types[project][region][type.name]["ecu"] = "Variable" else @@instance_types[project][region][type.name]["ecu"] = type.guest_cpus end } @@instance_types end
List all known Google
Cloud
Platform regions @param us_only [Boolean]: Restrict results to United States only
# File modules/mu/providers/google.rb, line 735 def self.listRegions(us_only = false, credentials: nil) if !MU::Cloud::Google.defaultProject(credentials) return [] end if @@regions.size == 0 begin result = MU::Cloud::Google.compute(credentials: credentials).list_regions(MU::Cloud::Google.defaultProject(credentials)) rescue ::Google::Apis::ClientError => e if e.message.match(/forbidden/) raise MuError, "Insufficient permissions to list Google Cloud region. The service account #{myServiceAccount} should probably have the project owner role." end raise e end result.items.each { |region| @@regions[region.name] = [] region.zones.each { |az| @@regions[region.name] << az.sub(/^.*?\/([^\/]+)$/, '\1') } } end if us_only @@regions.keys.delete_if { |r| !r.match(/^us/) } else @@regions.keys end end
Pull our global Google
Cloud
Platform credentials out of their secure vault, feed them to the googleauth gem, and stash the results on hand for consumption by the various GCP APIs. @param scopes [Array<String>]: One or more scopes for which to authorizer the caller. Will vary depending on the API you're calling.
# File modules/mu/providers/google.rb, line 534 def self.loadCredentials(scopes = nil, credentials: nil) if @@authorizers[credentials] and @@authorizers[credentials][scopes.to_s] return @@authorizers[credentials][scopes.to_s] end cfg = credConfig(credentials) if cfg if cfg['project'] @@enable_semaphores[cfg['project']] ||= Mutex.new end data = nil @@authorizers[credentials] ||= {} def self.get_machine_credentials(scopes, credentials = nil) @@svc_account_name = MU::Cloud::Google.getGoogleMetaData("instance/service-accounts/default/email") MU.log "We are hosted in GCP, so I will attempt to use the service account #{@@svc_account_name} to make API requests.", MU::DEBUG @@authorizers[credentials][scopes.to_s] = ::Google::Auth.get_application_default(scopes) @@authorizers[credentials][scopes.to_s].fetch_access_token! @@default_project ||= MU::Cloud::Google.getGoogleMetaData("project/project-id") begin listRegions(credentials: credentials) listInstanceTypes(credentials: credentials) listHabitats(credentials) rescue ::Google::Apis::ClientError MU.log "Found machine credentials #{@@svc_account_name}, but these don't appear to have sufficient permissions or scopes", MU::WARN, details: scopes @@authorizers.delete(credentials) return nil end @@authorizers[credentials][scopes.to_s] end if cfg["credentials_file"] or cfg["credentials_encoded"] begin data = if cfg["credentials_encoded"] JSON.parse(Base64.decode64(cfg["credentials_encoded"])) else JSON.parse(File.read(cfg["credentials_file"])) end @@default_project ||= data["project_id"] creds = { :json_key_io => StringIO.new(MultiJson.dump(data)), :scope => scopes } @@svc_account_name = data["client_email"] @@authorizers[credentials][scopes.to_s] = ::Google::Auth::ServiceAccountCredentials.make_creds(creds) return @@authorizers[credentials][scopes.to_s] rescue JSON::ParserError, Errno::ENOENT, Errno::EACCES => e if !MU::Cloud::Google.hosted? raise MuError, "Google Cloud credentials file #{cfg["credentials_file"]} is missing or invalid (#{e.message})" end MU.log "Google Cloud credentials file #{cfg["credentials_file"]} is missing or invalid", MU::WARN, details: e.message return get_machine_credentials(scopes, credentials) end elsif cfg["credentials"] begin vault, item = cfg["credentials"].split(/:/) data = MU::Groomer::Chef.getSecret(vault: vault, item: item).to_h rescue MU::Groomer::MuNoSuchSecret if !MU::Cloud::Google.hosted? raise MuError, "Google Cloud credentials not found in Vault #{vault}:#{item}" end MU.log "Google Cloud credentials not found in Vault #{vault}:#{item}", MU::WARN found = get_machine_credentials(scopes, credentials) raise MuError, "No valid credentials available! Either grant admin privileges to machine service account, or manually add a different one with mu-configure" if found.nil? return found end @@default_project ||= data["project_id"] creds = { :json_key_io => StringIO.new(MultiJson.dump(data)), :scope => scopes } @@svc_account_name = data["client_email"] @@authorizers[credentials][scopes.to_s] = ::Google::Auth::ServiceAccountCredentials.make_creds(creds) return @@authorizers[credentials][scopes.to_s] elsif MU::Cloud::Google.hosted? found = get_machine_credentials(scopes, credentials) raise MuError, "No valid credentials available! Either grant admin privileges to machine service account, or manually add a different one with mu-configure" if found.nil? return found else raise MuError, "Google Cloud credentials not configured" end end nil end
Google's StackDriver Logging Service API @param subclass [<Google::Apis::LoggingV2>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 973 def self.logging(subclass = nil, credentials: nil) require 'google/apis/logging_v2' if subclass.nil? @@logging_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "LoggingV2::LoggingService", scopes: ['cloud-platform'], credentials: credentials) return @@logging_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("LoggingV2").const_get(subclass) end end
If we've configured Google
as a provider, or are simply hosted in GCP, decide what our default region is.
# File modules/mu/providers/google.rb, line 322 def self.myRegion(credentials = nil) cfg = credConfig(credentials) if cfg and cfg['region'] @@myRegion_var = cfg['region'] elsif MU::Cloud::Google.hosted? zone = MU::Cloud::Google.getGoogleMetaData("instance/zone") @@myRegion_var = zone.gsub(/^.*?\/|\-\d+$/, "") else @@myRegion_var = "us-east4" end @@myRegion_var end
If this Mu master resides in the Google
Cloud
Platform, return the default service account associated with its metadata.
# File modules/mu/providers/google.rb, line 651 def self.myServiceAccount if MU::Cloud::Google.hosted? MU::Cloud::Google.getGoogleMetaData("instance/service-accounts/default/email") else nil end end
If we reside in this cloud, return the VPC
in which we, the Mu Master
, reside. @return [MU::Cloud::VPC]
# File modules/mu/providers/google.rb, line 131 def self.myVPCObj return nil if !hosted? instance = MU.myCloudDescriptor return nil if !instance or !instance.network_interfaces or instance.network_interfaces.size == 0 vpc = MU::MommaCat.findStray("Google", "vpc", cloud_id: instance.network_interfaces.first.network.gsub(/.*?\/([^\/]+)$/, '\1'), dummy_ok: true, habitats: [myProject]) return nil if vpc.nil? or vpc.size == 0 vpc.first end
Google
has fairly strict naming conventions (all lowercase, no underscores, etc). Provide a wrapper to our standard names to handle it.
# File modules/mu/providers/google.rb, line 799 def self.nameStr(name) name.downcase.gsub(/[^a-z0-9\-]/, "-") end
A shortcut for {MU::MommaCat.findStray} to resolve a shorthand project name into a cloud object, whether it refers to a sibling by internal name or by cloud identifier. @param name [String] @param deploy [String] @param raise_on_fail [Boolean] @param sibling_only [Boolean] @return [MU::Config::Habitat,nil]
# File modules/mu/providers/google.rb, line 238 def self.projectLookup(name, deploy = MU.mommacat, raise_on_fail: true, sibling_only: false) project_obj = deploy.findLitterMate(type: "habitats", name: name) if deploy and caller.grep(/`findLitterMate'/).empty? # XXX the dumbest if !project_obj and !sibling_only resp = MU::MommaCat.findStray( "Google", "habitats", deploy_id: deploy ? deploy.deploy_id : nil, cloud_id: name, name: name, dummy_ok: true ) project_obj = resp.first if resp and resp.size > 0 end if (!project_obj or !project_obj.cloud_id) and raise_on_fail raise MuError, "Failed to find project '#{name}' in deploy #{deploy.deploy_id}" end project_obj end
Take a plain string that might be a reference to sibling project declared elsewhere in the active stack, or the project id of a live cloud resource, and return a {MU::Config::Ref} object @param project [String]: The name of a sibling project, or project id of an active project in GCP @param config [MU::Config]: A {MU::Config} object containing sibling resources, typically what we'd pass if we're calling during configuration parsing @param credentials [String]: @return [MU::Config::Ref]
# File modules/mu/providers/google.rb, line 196 def self.projectToRef(project, config: nil, credentials: nil) return nil if !project if config and config.haveLitterMate?(project, "habitat") ref = MU::Config::Ref.new( name: project, cloud: "Google", credentials: credentials, type: "habitats" ) end if !ref resp = MU::MommaCat.findStray( "Google", "habitats", cloud_id: project, credentials: credentials, dummy_ok: true ) if resp and resp.size > 0 project_obj = resp.first ref = MU::Config::Ref.new( id: project_obj.cloud_id, cloud: "Google", credentials: credentials, type: "habitats" ) end end ref end
Remove the service account and various deploy secrets associated with a deployment. Intended for invocation from MU::Cleanup
. @param deploy_id [String]: The deploy for which we're granting the secret @param noop [Boolean]: If true, will only print what would be done
# File modules/mu/providers/google.rb, line 378 def self.removeDeploySecretsAndRoles(deploy_id = MU.deploy_id, flags: {}, noop: false, credentials: nil) cfg = credConfig(credentials) return if !cfg or !cfg['project'] flags["project"] ||= cfg['project'] resp = MU::Cloud::Google.storage(credentials: credentials).list_objects( adminBucketName(credentials), prefix: deploy_id ) if resp and resp.items resp.items.each { |obj| MU.log "Deleting gs://#{adminBucketName(credentials)}/#{obj.name}" if !noop MU::Cloud::Google.storage(credentials: credentials).delete_object( adminBucketName(credentials), obj.name ) end } end end
Any cloud-specific instance methods we require our resource implementations to have, above and beyond the ones specified by {MU::Cloud} @return [Array<Symbol>]
# File modules/mu/providers/google.rb, line 51 def self.required_instance_methods [:url] end
A hook that is always called just before any of the instance method of our resource implementations gets invoked, so that we can ensure that repetitive setup tasks (like resolving :resource_group
for Azure
resources) have always been done. @param cloudobj [MU::Cloud] @param deploy [MU::MommaCat]
# File modules/mu/providers/google.rb, line 77 def self.resourceInitHook(cloudobj, deploy) class << self attr_reader :project_id attr_reader :customer # url is too complex for an attribute (we get it from the cloud API), # so it's up in AdditionalResourceMethods instead end return if !cloudobj cloudobj.instance_variable_set(:@customer, MU::Cloud::Google.customerID(cloudobj.config['credentials'])) # XXX ensure @cloud_id and @project_id if this is a habitat # XXX skip project_id if this is a folder or group if deploy # XXX this may be wrong for new deploys (but def right for regrooms) project = MU::Cloud::Google.projectLookup(cloudobj.config['project'], deploy, sibling_only: true, raise_on_fail: false) project_id = project.nil? ? cloudobj.config['project'] : project.cloudobj.cloud_id cloudobj.instance_variable_set(:@project_id, project_id) else cloudobj.instance_variable_set(:@project_id, cloudobj.config['project']) end # XXX @url? Well we're not likely to have @cloud_desc at this point, so maybe # that needs to be a generic-to-google wrapper like def url; cloud_desc.self_link;end # XXX something like: vpc["habitat"] = MU::Cloud::Google.projectToRef(vpc["project"], config: configurator, credentials: vpc["credentials"]) end
Google's Cloud
Resource Manager API @param subclass [<Google::Apis::CloudresourcemanagerV1>]: If specified, will return the class ::Google::Apis::CloudresourcemanagerV1::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 892 def self.resource_manager(subclass = nil, credentials: nil) require 'google/apis/cloudresourcemanager_v1' if subclass.nil? if !MU::Cloud::Google.credConfig(credentials) raise MuError, "No such credential set #{credentials} defined in mu.yaml!" end @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as']) return @@resource_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV1").const_get(subclass) end end
Google's Service Manager API (the one you use to enable pre-project APIs) @param subclass [<Google::Apis::ServicemanagementV1>]: If specified, will return the class ::Google::Apis::ServicemanagementV1::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 934 def self.service_manager(subclass = nil, credentials: nil) require 'google/apis/servicemanagement_v1' if subclass.nil? @@service_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "ServicemanagementV1::ServiceManagementService", scopes: ['cloud-platform'], credentials: credentials) return @@service_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("ServicemanagementV1").const_get(subclass) end end
Google's SQL Service API @param subclass [<Google::Apis::SqladminV1beta4>]: If specified, will return the class ::Google::Apis::SqladminV1beta4::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 947 def self.sql(subclass = nil, credentials: nil) require 'google/apis/sqladmin_v1beta4' if subclass.nil? @@sql_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "SqladminV1beta4::SQLAdminService", scopes: ['cloud-platform'], credentials: credentials) return @@sql_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("SqladminV1beta4").const_get(subclass) end end
Google's Storage Service API @param subclass [<Google::Apis::StorageV1>]: If specified, will return the class ::Google::Apis::StorageV1::subclass instead of an API client instance
# File modules/mu/providers/google.rb, line 833 def self.storage(subclass = nil, credentials: nil) require 'google/apis/storage_v1' if subclass.nil? @@storage_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "StorageV1::StorageService", scopes: ['cloud-platform'], credentials: credentials) return @@storage_api[credentials] elsif subclass.is_a?(Symbol) return Object.const_get("::Google").const_get("Apis").const_get("StorageV1").const_get(subclass) end end
Fetch the name of the service account we were using last time we loaded GCP credentials. @return [String]
# File modules/mu/providers/google.rb, line 527 def self.svc_account_name @@svc_account_name end
Is this a “real” cloud provider, or a stub like CloudFormation
?
# File modules/mu/providers/google.rb, line 56 def self.virtual? false end
Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it @param deploy_id [String]: The deploy for which we're writing the secret @param value [String]: The contents of the secret
# File modules/mu/providers/google.rb, line 351 def self.writeDeploySecret(deploy, value, name = nil, credentials: nil) deploy_id = deploy.deploy_id name ||= deploy_id+"-secret" begin MU.log "Writing #{name} to Cloud Storage bucket #{adminBucketName(credentials)}" f = Tempfile.new(name) # XXX this is insecure and stupid f.write value f.close objectobj = MU::Cloud::Google.storage(:Object).new( bucket: adminBucketName(credentials), name: name ) MU::Cloud::Google.storage(credentials: credentials).insert_object( adminBucketName(credentials), objectobj, upload_source: f.path ) f.unlink rescue ::Google::Apis::ClientError => e raise MU::MommaCat::DeployInitializeError, "Got #{e.inspect} trying to write #{name} to #{adminBucketName(credentials)}" end end