class Naginegi::BigQuery
Constants
- CONTENTS
Public Class Methods
generate_schema(columns)
click to toggle source
# File lib/naginegi/bigquery.rb, line 47 def self.generate_schema(columns) json_body = columns.map(&:to_json).join(",\n") "[\n" + json_body + "\n]\n" end
generate_sql(table_config, columns)
click to toggle source
# File lib/naginegi/bigquery.rb, line 52 def self.generate_sql(table_config, columns) columns = columns.map(&:converted_value) sql = "SELECT #{columns.join(',')}" sql << " FROM #{table_config.name}" sql << " WHERE #{table_config.condition}" if table_config.condition sql << "\n" sql end
new(config)
click to toggle source
# File lib/naginegi/bigquery.rb, line 42 def initialize(config) @config = config.dup @current_date = Date.today end
Public Instance Methods
actual_table_name(table_name, daily_snapshot)
click to toggle source
# File lib/naginegi/bigquery.rb, line 111 def actual_table_name(table_name, daily_snapshot) return table_name unless daily_snapshot table_name + @current_date.strftime('%Y%m%d') end
delete_table(dataset, table_name)
click to toggle source
# File lib/naginegi/bigquery.rb, line 95 def delete_table(dataset, table_name) keyfile = if @config['json_key'] value = @config['json_key'].dup value['private_key'] = value['private_key'].gsub('\\n', "\n") value else @config['json_keyfile'] end bq = Google::Cloud::Bigquery.new( project: @config['project_id'], keyfile: keyfile ) bq.service.delete_table(dataset, table_name) end
generate_embulk_config(db_name, db_config, table_config, columns)
click to toggle source
# File lib/naginegi/bigquery.rb, line 61 def generate_embulk_config(db_name, db_config, table_config, columns) db_type = db_config['db_type'] host = db_config['host'] user = db_config['username'] password = db_config['password'] database = db_config['database'] ssl = db_config['embulk_ssl_enable'] || false options = if db_type == 'mysql' "options: {useLegacyDatetimeCode: false, serverTimezone: #{db_config['timezone']}}" else '' end query = Naginegi::BigQuery.generate_sql(table_config, columns) auth_method = @config['auth_method'] if @config['json_key'] values = @config['json_key'].map do |k, v| value = v.gsub("\n", '\\n') "\"#{k}\": \"#{value}\"" end json_key_content = "content: |\n {#{values.join(',')}}" else json_keyfile = @config['json_keyfile'] end project = @config['project_id'] service_account_email = @config['service_email'] dataset = db_config['bq_dataset'] table_name = actual_table_name(table_config.name, db_config['daily_snapshot'] || table_config.daily_snapshot) schema_file = "#{@config['schema_dir']}/#{db_name}/#{table_config.name}.json" path_prefix = "/var/tmp/embulk_#{db_name}_#{table_config.name}" ERB.new(CONTENTS).result(binding) end