class MU::Cloud::Google::GoogleEndpoint
Wrapper class for Google
APIs, so that we can catch some common transient endpoint errors without having to spray rescues all over the codebase.
Attributes
Public Class Methods
Create a Google
Cloud
Platform API client @param api [String]: Which API are we wrapping? @param scopes [Array<String>]: Google
auth scopes applicable to this API
# File modules/mu/providers/google.rb, line 1113 def initialize(api: "ComputeV1::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], masquerade: nil, credentials: nil, auth_error_quiet: false) @credentials = credentials @scopes = scopes.map { |s| if !s.match(/\//) # allow callers to use shorthand s = "https://www.googleapis.com/auth/"+s end s } @masquerade = masquerade @api = Object.const_get("Google::Apis::#{api}").new @api.authorization = MU::Cloud::Google.loadCredentials(@scopes, credentials: credentials) raise MuError, "No useable Google credentials found#{credentials ? " with set '#{credentials}'" : ""}" if @api.authorization.nil? if @masquerade begin @api.authorization.sub = @masquerade @api.authorization.fetch_access_token! rescue Signet::AuthorizationError => e if auth_error_quiet MU.log "Cannot masquerade as #{@masquerade} to API #{api}: #{e.message}", MU::DEBUG, details: @scopes else MU.log "Cannot masquerade as #{@masquerade} to API #{api}: #{e.message}", MU::ERROR, details: @scopes if e.message.match(/client not authorized for any of the scopes requested/) # XXX it'd be helpful to list *all* scopes we like, as well as the API client's numeric id MU.log "To grant access to API scopes for this service account, see:", MU::ERR, details: "https://admin.google.com/AdminHome?chromeless=1#OGX:ManageOauthClients" end end raise e end end @issuer = @api.authorization.issuer end
Public Instance Methods
Generic wrapper for deleting Compute resources, which are consistent enough that we can get away with this. @param type [String]: The type of resource, typically the string you'll find in all of the API calls referring to it @param project [String]: The project in which we should look for the resources @param region [String]: The region in which to loop for the resources @param noop [Boolean]: If true, will only log messages about resources to be deleted, without actually deleting them @param filter [String]: The Compute API filter string to use to isolate appropriate resources
# File modules/mu/providers/google.rb, line 1153 def delete(type, project, region = nil, noop = false, filter = "description eq #{MU.deploy_id}", credentials: nil) list_sym = "list_#{type.sub(/y$/, "ie")}s".to_sym credentials ||= @credentials resp = nil begin if region resp = MU::Cloud::Google.compute(credentials: credentials).send(list_sym, project, region, filter: filter, mu_gcp_enable_apis: false) else resp = MU::Cloud::Google.compute(credentials: credentials).send(list_sym, project, filter: filter, mu_gcp_enable_apis: false) end rescue ::Google::Apis::ClientError => e return if e.message.match(/^notFound: /) end if !resp.nil? and !resp.items.nil? threads = [] parent_thread_id = Thread.current.object_id resp.items.each { |obj| threads << Thread.new { MU.dupGlobals(parent_thread_id) Thread.abort_on_exception = false MU.log "Removing #{type.gsub(/_/, " ")} #{obj.name}" delete_sym = "delete_#{type}".to_sym if !noop retries = 0 failed = false begin resp = nil failed = false if region resp = MU::Cloud::Google.compute(credentials: credentials).send(delete_sym, project, region, obj.name) else resp = MU::Cloud::Google.compute(credentials: credentials).send(delete_sym, project, obj.name) end if resp.error and resp.error.errors and resp.error.errors.size > 0 failed = true retries += 1 if resp.error.errors.first.code == "RESOURCE_IN_USE_BY_ANOTHER_RESOURCE" and retries < 6 sleep 10 else MU.log "Error deleting #{type.gsub(/_/, " ")} #{obj.name}", MU::ERR, details: resp.error.errors Thread.abort_on_exception = false raise MuError, "Failed to delete #{type.gsub(/_/, " ")} #{obj.name}" end else failed = false end # TODO validate that the resource actually went away, because it seems not to do so very reliably rescue ::Google::Apis::ClientError => e raise e if !e.message.match(/(^notFound: |operation in progress)/) rescue MU::Cloud::MuDefunctHabitat => e # this is ok- it's already deleted end while failed and retries < 6 end } } threads.each do |t| t.join end end end
Check whether the various types of Operation
responses say they're done, without knowing which specific API they're from
# File modules/mu/providers/google.rb, line 1376 def is_done?(retval) (retval.respond_to?(:status) and retval.status == "DONE") or (retval.respond_to?(:done) and retval.done) end
Catch-all for AWS
client methods. Essentially a pass-through with some rescues for known silly endpoint behavior.
# File modules/mu/providers/google.rb, line 1221 def method_missing(method_sym, *arguments) retries = 0 actual_resource = nil enable_on_fail = true arguments.each { |arg| if arg.is_a?(Hash) and arg.has_key?(:mu_gcp_enable_apis) enable_on_fail = arg[:mu_gcp_enable_apis] arg.delete(:mu_gcp_enable_apis) end } arguments.delete({}) next_page_token = nil overall_retval = nil begin MU.log "Calling #{method_sym}", MU::DEBUG, details: arguments retval = nil retries = 0 wait_backoff = 5 if next_page_token if method_sym != :list_entry_log_entries if arguments.size == 1 and arguments.first.is_a?(Hash) arguments[0][:page_token] = next_page_token else arguments << { :page_token => next_page_token } end elsif arguments.first.class == ::Google::Apis::LoggingV2::ListLogEntriesRequest arguments[0] = ::Google::Apis::LoggingV2::ListLogEntriesRequest.new( resource_names: arguments.first.resource_names, filter: arguments.first.filter, page_token: next_page_token ) end end begin if !arguments.nil? and arguments.size == 1 retval = @api.method(method_sym).call(arguments[0]) elsif !arguments.nil? and arguments.size > 0 retval = @api.method(method_sym).call(*arguments) else retval = @api.method(method_sym).call end rescue ArgumentError => e MU.log "#{e.class.name} calling #{@api.class.name}.#{method_sym.to_s}: #{e.message}", MU::ERR, details: arguments raise e rescue ::Google::Apis::AuthorizationError => e if arguments.size > 0 raise MU::MuError, "Service account #{MU::Cloud::Google.svc_account_name} has insufficient privileges to call #{method_sym} in project #{arguments.first}" else raise MU::MuError, "Service account #{MU::Cloud::Google.svc_account_name} has insufficient privileges to call #{method_sym}" end rescue ::Google::Apis::RateLimitError, ::Google::Apis::TransmissionError, ::ThreadError, ::Google::Apis::ServerError => e if retries <= 10 sleep wait_backoff retries += 1 wait_backoff = wait_backoff * 2 retry else raise e end rescue ::Google::Apis::ClientError, OpenSSL::SSL::SSLError => e if e.message.match(/^quotaExceeded: Request rate/) if retries <= 10 sleep wait_backoff retries += 1 wait_backoff = wait_backoff * 2 retry else raise e end elsif e.message.match(/^invalidParameter:|^badRequest:/) MU.log "#{e.class.name} calling #{@api.class.name}.#{method_sym.to_s}: "+e.message, MU::ERR, details: arguments # uncomment for debugging stuff; this can occur in benign situations so we don't normally want it logging elsif e.message.match(/^forbidden:/) MU.log "#{e.class.name} calling #{@api.class.name}.#{method_sym.to_s} got \"#{e.message}\" using credentials #{@credentials}#{@masquerade ? " (OAuth'd as #{@masquerade})": ""}.#{@scopes ? "\nScopes:\n#{@scopes.join("\n")}" : "" }", MU::DEBUG, details: arguments raise e end @@enable_semaphores ||= {} max_retries = 3 wait_time = 90 if enable_on_fail and retries <= max_retries and e.message.match(/^accessNotConfigured/) enable_obj = nil project = if arguments.size > 0 and arguments.first.is_a?(String) arguments.first else MU::Cloud::Google.defaultProject(@credentials) end # XXX validate that this actually looks like a project id, maybe if method_sym == :delete and !MU::Cloud::Google::Habitat.isLive?(project, @credentials) MU.log "Got accessNotConfigured while attempting to delete a resource in #{project}", MU::WARN return end @@enable_semaphores[project] ||= Mutex.new enable_obj = MU::Cloud::Google.service_manager(:EnableServiceRequest).new( consumer_id: "project:"+project.gsub(/^projects\/([^\/]+)\/.*/, '\1') ) # XXX dumbass way to get this string if e.message.match(/by visiting https:\/\/console\.developers\.google\.com\/apis\/api\/(.+?)\//) svc_name = Regexp.last_match[1] save_verbosity = MU.verbosity if !["servicemanagement.googleapis.com", "billingbudgets.googleapis.com"].include?(svc_name) and method_sym != :delete retries += 1 @@enable_semaphores[project].synchronize { MU.setLogging(MU::Logger::NORMAL) MU.log "Attempting to enable #{svc_name} in project #{project.gsub(/^projects\/([^\/]+)\/.*/, '\1')}; will retry #{method_sym.to_s} in #{(wait_time/retries).to_s}s (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE MU.setLogging(save_verbosity) begin MU::Cloud::Google.service_manager(credentials: @credentials).enable_service(svc_name, enable_obj) rescue ::Google::Apis::ClientError => e MU.log "Error enabling #{svc_name} in #{project.gsub(/^projects\/([^\/]+)\/.*/, '\1')} for #{method_sym.to_s}: "+ e.message, MU::ERR, details: enable_obj raise e end } sleep wait_time/retries retry else MU.setLogging(MU::Logger::NORMAL) MU.log "Google Cloud's Service Management API must be enabled manually by visiting #{e.message.gsub(/.*?(https?:\/\/[^\s]+)(?:$|\s).*/, '\1')}", MU::ERR MU.setLogging(save_verbosity) raise MU::MuError, "Service Management API not yet enabled for this account/project" end elsif e.message.match(/scheduled for deletion and cannot be used for API calls/) raise MuDefunctHabitat, e.message else MU.log "Unfamiliar error calling #{method_sym.to_s} "+e.message, MU::ERR, details: arguments end elsif retries <= 10 and e.message.match(/^resourceNotReady:/) or (e.message.match(/^resourceInUseByAnotherResource:/) and method_sym.to_s.match(/^delete_/)) or e.message.match(/SSL_connect/) if retries > 0 and retries % 3 == 0 MU.log "Will retry #{method_sym} after #{e.message} (retry #{retries})", MU::NOTICE, details: arguments else MU.log "Will retry #{method_sym} after #{e.message} (retry #{retries})", MU::DEBUG, details: arguments end retries = retries + 1 sleep retries*10 retry else raise e end end if retval.class.name.match(/.*?::Operation$/) retries = 0 # Check whether the various types of +Operation+ responses say # they're done, without knowing which specific API they're from def is_done?(retval) (retval.respond_to?(:status) and retval.status == "DONE") or (retval.respond_to?(:done) and retval.done) end begin if retries > 0 and retries % 3 == 0 MU.log "Waiting for #{method_sym} to be done (retry #{retries})", MU::NOTICE else MU.log "Waiting for #{method_sym} to be done (retry #{retries})", MU::DEBUG, details: retval end if !is_done?(retval) sleep 7 begin if retval.class.name.match(/::Compute[^:]*::/) resp = MU::Cloud::Google.compute(credentials: @credentials).get_global_operation( arguments.first, # there's always a project id retval.name ) retval = resp elsif retval.class.name.match(/::Servicemanagement[^:]*::/) resp = MU::Cloud::Google.service_manager(credentials: @credentials).get_operation( retval.name ) retval = resp elsif retval.class.name.match(/::Cloudresourcemanager[^:]*::/) resp = MU::Cloud::Google.resource_manager(credentials: @credentials).get_operation( retval.name ) retval = resp if retval.error raise MuError, retval.error.message end elsif retval.class.name.match(/::Container[^:]*::/) resp = MU::Cloud::Google.container(credentials: @credentials).get_project_location_operation( retval.self_link.sub(/.*?\/projects\//, 'projects/') ) retval = resp elsif retval.class.name.match(/::Cloudfunctions[^:]*::/) resp = MU::Cloud::Google.function(credentials: @credentials).get_operation( retval.name ) retval = resp #MU.log method_sym.to_s, MU::WARN, details: retval if retval.error raise MuError, retval.error.message end else pp retval raise MuError, "I NEED TO IMPLEMENT AN OPERATION HANDLER FOR #{retval.class.name}" end rescue ::Google::Apis::ClientError => e # this is ok; just means the operation is done and went away if e.message.match(/^notFound:/) break else raise e end end retries = retries + 1 end end while !is_done?(retval) # Most insert methods have a predictable get_* counterpart. Let's # take advantage. # XXX might want to do something similar for delete ops? just the # but where we wait for the operation to definitely be done # had_been_found = false if method_sym.to_s.match(/^(insert|create|patch)_/) get_method = method_sym.to_s.gsub(/^(insert|patch|create_disk|create)_/, "get_").to_sym cloud_id = if retval.respond_to?(:target_link) retval.target_link.sub(/^.*?\/([^\/]+)$/, '\1') elsif retval.respond_to?(:metadata) and retval.metadata["target"] retval.metadata["target"] else arguments[0] # if we're lucky end faked_args = arguments.dup faked_args.pop if get_method == :get_snapshot faked_args.pop faked_args.pop end faked_args.push(cloud_id) if get_method == :get_project_location_cluster faked_args[0] = faked_args[0]+"/clusters/"+faked_args[1] faked_args.pop elsif get_method == :get_project_location_function faked_args = [cloud_id] end actual_resource = @api.method(get_method).call(*faked_args) #if method_sym == :insert_instance #MU.log "actual_resource", MU::WARN, details: actual_resource #end # had_been_found = true if actual_resource.respond_to?(:status) and ["PROVISIONING", "STAGING", "PENDING", "CREATING", "RESTORING"].include?(actual_resource.status) retries = 0 begin if retries > 0 and retries % 3 == 0 MU.log "Waiting for #{cloud_id} to get past #{actual_resource.status} (retry #{retries})", MU::NOTICE else MU.log "Waiting for #{cloud_id} to get past #{actual_resource.status} (retry #{retries})", MU::DEBUG, details: actual_resource end sleep 10 actual_resource = @api.method(get_method).call(*faked_args) retries = retries + 1 end while ["PROVISIONING", "STAGING", "PENDING", "CREATING", "RESTORING"].include?(actual_resource.status) end return actual_resource end end # This atrocity appends the pages of list_* results if overall_retval if method_sym.to_s.match(/^list_(.*)/) require 'google/apis/iam_v1' require 'google/apis/logging_v2' what = Regexp.last_match[1].to_sym whatassign = (Regexp.last_match[1]+"=").to_sym if overall_retval.class == ::Google::Apis::IamV1::ListServiceAccountsResponse what = :accounts whatassign = :accounts= end if retval.respond_to?(what) and retval.respond_to?(whatassign) if !retval.public_send(what).nil? newarray = retval.public_send(what) + overall_retval.public_send(what) overall_retval.public_send(whatassign, newarray) end elsif !retval.respond_to?(:next_page_token) or retval.next_page_token.nil? or retval.next_page_token.empty? MU.log "Not sure how to append #{method_sym.to_s} results to #{overall_retval.class.name} (apparently #{what.to_s} and #{whatassign.to_s} aren't it), returning first page only", MU::WARN, details: retval return retval end else MU.log "Not sure how to append #{method_sym.to_s} results, returning first page only", MU::WARN, details: retval return retval end else overall_retval = retval end arguments.delete({ :page_token => next_page_token }) next_page_token = nil if retval.respond_to?(:next_page_token) and !retval.next_page_token.nil? next_page_token = retval.next_page_token MU.log "Getting another page of #{method_sym.to_s}", MU::DEBUG, details: next_page_token else return overall_retval end rescue ::Google::Apis::ServerError, ::Google::Apis::ClientError, ::Google::Apis::TransmissionError => e if e.class.name == "Google::Apis::ClientError" and (!method_sym.to_s.match(/^insert_/) or !e.message.match(/^notFound: /) or (e.message.match(/^notFound: /) and method_sym.to_s.match(/^insert_/)) ) if e.message.match(/^notFound: /) and method_sym.to_s.match(/^insert_/) and retval logreq = MU::Cloud::Google.logging(:ListLogEntriesRequest).new( resource_names: ["projects/"+arguments.first], filter: %Q{labels."compute.googleapis.com/resource_id"="#{retval.target_id}" OR labels."ssl_certificate_id"="#{retval.target_id}"} # XXX I guess we need to cover all of the possible keys, ugh ) logs = MU::Cloud::Google.logging(credentials: @credentials).list_entry_log_entries(logreq) details = nil if logs.entries details = logs.entries.map { |err| err.json_payload } details.reject! { |err| err["error"].nil? or err["error"].size == 0 } end raise MuError, "#{method_sym.to_s} of #{retval.target_id} appeared to succeed, but then the resource disappeared! #{details.to_s}" end raise e end retries = retries + 1 debuglevel = MU::DEBUG interval = 5 + Random.rand(4) - 2 if retries < 10 and retries > 2 debuglevel = MU::NOTICE interval = 20 + Random.rand(10) - 3 # elsif retries >= 10 and retries <= 100 elsif retries >= 10 debuglevel = MU::WARN interval = 40 + Random.rand(15) - 5 # elsif retries > 100 # raise MuError, "Exhausted retries after #{retries} attempts while calling Compute's #{method_sym} in #{@region}. Args were: #{arguments}" end MU.log "Got #{e.inspect} calling Google's #{method_sym}, waiting #{interval.to_s}s and retrying. Called from: #{caller[1]}", debuglevel, details: arguments sleep interval MU.log method_sym.to_s.bold+" "+e.inspect, MU::WARN, details: arguments retry end while !next_page_token.nil? end