module ExportToGcloud

Constants

VERSION

Public Class Methods

client() click to toggle source
# File lib/export_to_gcloud.rb, line 12
def self.client
  @client || raise('Gcloud client not present. call ExportToGcloud#setup first.')
end
create_context(**opts) click to toggle source
# File lib/export_to_gcloud/library.rb, line 43
def self.create_context **opts
  ::ExportToGcloud::Exporter::Context.new client, opts
end
definitions_resolver=(proc) click to toggle source
# File lib/export_to_gcloud/library.rb, line 4
def self.definitions_resolver= proc
  @definitions_resolver = proc
end
get_exporter(name, context) click to toggle source
# File lib/export_to_gcloud/library.rb, line 31
def self.get_exporter name, context
  name = name.to_s

  @definitions ||= {}
  unless @definitions.has_key? name
    @definitions[name] = ::ExportToGcloud::Exporter::Definition.load_definition name, @definitions_resolver
  end

  definition = @definitions[name]
  definition.type.new definition, context
end
setup(project_name:, config_file:, definitions_resolver:nil) click to toggle source
# File lib/export_to_gcloud.rb, line 5
def self.setup project_name:, config_file:, definitions_resolver:nil
  require_relative 'export_to_gcloud/library'

  self.definitions_resolver = definitions_resolver if definitions_resolver
  @client = ::Gcloud.new project_name, config_file
end
wait_for_load_jobs(jobs, &block) click to toggle source

waits for BigQuery jobs

  • send a block to do something with failed

# File lib/export_to_gcloud/library.rb, line 10
def self.wait_for_load_jobs(jobs, &block)
  jobs_left = jobs.dup
  failed = []
  sleeper = ->(_retries) {sleep 2 * _retries + 5}
  retries = 0

  until jobs_left.empty?
    sleeper.call retries
    retries += 1
    jobs_left.each &:reload!
    jobs_left.delete_if do |j|
      if j.done?
        failed << {id: j.job_id, error: j.error, sources: j.sources} if j.failed?
        true
      end
    end
  end

  block.call failed unless failed.empty?
end