class Naginegi::BigQuery

Constants

CONTENTS

Public Class Methods

generate_schema(columns) click to toggle source
# File lib/naginegi/bigquery.rb, line 47
def self.generate_schema(columns)
  json_body = columns.map(&:to_json).join(",\n")
  "[\n" + json_body + "\n]\n"
end
generate_sql(table_config, columns) click to toggle source
# File lib/naginegi/bigquery.rb, line 52
def self.generate_sql(table_config, columns)
  columns = columns.map(&:converted_value)
  sql = "SELECT #{columns.join(',')}"
  sql << " FROM #{table_config.name}"
  sql << " WHERE #{table_config.condition}" if table_config.condition
  sql << "\n"
  sql
end
new(config) click to toggle source
# File lib/naginegi/bigquery.rb, line 42
def initialize(config)
  @config = config.dup
  @current_date = Date.today
end

Public Instance Methods

actual_table_name(table_name, daily_snapshot) click to toggle source
# File lib/naginegi/bigquery.rb, line 111
def actual_table_name(table_name, daily_snapshot)
  return table_name unless daily_snapshot
  table_name + @current_date.strftime('%Y%m%d')
end
delete_table(dataset, table_name) click to toggle source
# File lib/naginegi/bigquery.rb, line 95
def delete_table(dataset, table_name)
  keyfile = if @config['json_key']
              value = @config['json_key'].dup
              value['private_key'] = value['private_key'].gsub('\\n', "\n")
              value
            else
              @config['json_keyfile']
            end

  bq = Google::Cloud::Bigquery.new(
    project: @config['project_id'],
    keyfile: keyfile
  )
  bq.service.delete_table(dataset, table_name)
end
generate_embulk_config(db_name, db_config, table_config, columns) click to toggle source
# File lib/naginegi/bigquery.rb, line 61
def generate_embulk_config(db_name, db_config, table_config, columns)
  db_type = db_config['db_type']
  host = db_config['host']
  user = db_config['username']
  password = db_config['password']
  database = db_config['database']
  ssl = db_config['embulk_ssl_enable'] || false
  options = if db_type == 'mysql'
              "options: {useLegacyDatetimeCode: false, serverTimezone: #{db_config['timezone']}}"
            else
              ''
            end
  query = Naginegi::BigQuery.generate_sql(table_config, columns)

  auth_method = @config['auth_method']
  if @config['json_key']
    values = @config['json_key'].map do |k, v|
      value = v.gsub("\n", '\\n')
      "\"#{k}\": \"#{value}\""
    end
    json_key_content = "content: |\n      {#{values.join(',')}}"
  else
    json_keyfile = @config['json_keyfile']
  end
  project = @config['project_id']
  service_account_email = @config['service_email']
  dataset = db_config['bq_dataset']
  table_name = actual_table_name(table_config.name, db_config['daily_snapshot'] || table_config.daily_snapshot)
  schema_file = "#{@config['schema_dir']}/#{db_name}/#{table_config.name}.json"
  path_prefix = "/var/tmp/embulk_#{db_name}_#{table_config.name}"

  ERB.new(CONTENTS).result(binding)
end