class Ckancli::Commands::Upload
Public Class Methods
new(options)
click to toggle source
# File lib/ckancli/commands/upload.rb, line 20 def initialize(options) @options = options @dir = nil @resources = [] @http_resource = false @schema = nil @resource_config = nil @dataset_config = nil @api_config = nil @logger = nil @ckan_client = nil @update_modified = false @mail_server = nil end
Public Instance Methods
execute(input: $stdin, output: $stdout)
click to toggle source
# File lib/ckancli/commands/upload.rb, line 36 def execute(input: $stdin, output: $stdout) begin prepare_options(@options) if !@schema.nil? validate() else puts "" show_info "No schema - skipping validation" end upload() if !@dataset_config.nil? update_dataset() else puts "" show_info "No dataset configuration - skipping update" end @resources.each do |res| if !res[:log_file].nil? res[:log_file].close end end if !@mail_server.nil? && !@mail_receivers.nil? mail_notify() end rescue StandardError => e return_error "Unexpected error", e end end
Private Instance Methods
format_validation(error, schema = nil)
click to toggle source
# File lib/ckancli/commands/upload.rb, line 421 def format_validation(error, schema = nil) h = { type: error.type, category: error.category, row: error.row, col: error.column } if error.column && !schema.nil? && schema.class == Csvlint::Schema && schema.fields[error.column - 1] != nil field = schema.fields[error.column - 1] h[:header] = field.name end h end
log_error(message)
click to toggle source
# File lib/ckancli/commands/upload.rb, line 449 def log_error(message) if !@logger.nil? @logger.error message end end
log_fatal(message, exception = nil)
click to toggle source
# File lib/ckancli/commands/upload.rb, line 437 def log_fatal(message, exception = nil) if !@logger.nil? @logger.fatal message, exception end end
log_info(message)
click to toggle source
# File lib/ckancli/commands/upload.rb, line 443 def log_info(message) if !@logger.nil? @logger.info message end end
mail_notify()
click to toggle source
# File lib/ckancli/commands/upload.rb, line 237 def mail_notify puts "" show_info "Sending e-mail notifications..." begin mail = Mail.new mail[:from] = @mail_server["sender"] mail[:subject] = @mail_server["subject"] mail.delivery_method :smtp, { :address => @mail_server["address"], :port => @mail_server["port"], :user_name => @mail_server["smtp_user"], :password => @mail_server["smtp_password"], :authentication => @mail_server["smtp_user"].nil? ? "none" : "plain", :ssl => @mail_server["ssl"], :openssl_verify_mode => OpenSSL::SSL::VERIFY_NONE } # message msg = "CKAN CLI task completed.\r\n\r\n#{@resources.length} files processed" has_errors = false # attachments @resources.each do |res| msg = "#{msg}\r\n- #{File.basename(res[:path])}" if !res[:error].nil? msg = "#{msg} (#{res[:error]})" has_errors = true elsif !res[:summary].nil? msg = "#{msg} (#{res[:summary][:errors].length} validation errors, #{res[:summary][:warnings].length} validation warnings)" if res[:summary][:errors].length > 0 has_errors = true end end mail.add_file :filename => File.basename(res[:path] + ".log"), :content => File.read(res[:path] + ".log") end msg = "#{msg}\r\n\r\nLog files attached." mail[:body] = msg mail[:to] = has_errors ? @mail_receivers["error"] : @mail_receivers["success"] mail.deliver! rescue StandardError => e show_error "Could not send e-mail", e end show_info "End of sending" end
prepare_options(options)
click to toggle source
# File lib/ckancli/commands/upload.rb, line 71 def prepare_options(options) @http_resource = true if options[:dir] =~ /^http(s)?/ if @http_resource @dir = Dir.pwd else @dir = File.directory?(options[:dir]) ? options[:dir] : (File.file?(options[:dir]) ? File.dirname(options[:dir]) : nil) end if @dir.nil? return_error "Invalid directory - #{options[:dir]}" end begin @logger = TTY::Logger.new do |config| config.metadata = [:date, :time] config.handlers = [ [:stream, output: File.open(File.join(@dir, "output.log"), "a")] ] end rescue StandardError => e return_error "Could not initialize logger", e end begin if @http_resource file_path = File.join(@dir, File.basename(URI.parse(options[:dir]).path)) File.open(file_path, "wb") do |file| file.write open(options[:dir]).read end @resources.push(file_path) else if File.directory?(options[:dir]) @resources = Dir.entries(options[:dir]).select{ |f| File.file?(File.join(options[:dir], f)) && (options[:ignoreextension] || File.extname(f).downcase == ".csv") }.map { |f| File.join(options[:dir], f) } else @resources.push(options[:dir]) end end @resources = @resources.map{ |r| { :path => r, :valid => true, :summary => nil, :error => nil, :log_file => File.open(File.join(@dir, "#{File.basename(r)}.log"), "w"), :log => nil } } @resources.each do |res| res[:log] = TTY::Logger.new do |config| config.metadata = [:date, :time] config.handlers = [ [:stream, output: res[:log_file]] ] end end rescue Errno::ENOENT => e return_error "Could not load resources", e rescue StandardError => e return_error "Could not load resources", e end if options[:validationschema] @schema = (Pathname.new(options[:validationschema])).absolute? ? options[:validationschema] : File.join(@dir, options[:validationschema]) end if options[:resourceconfig] begin @resource_config = JSON.parse(File.read((Pathname.new(options[:resourceconfig])).absolute? ? options[:resourceconfig] : File.join(@dir, options[:resourceconfig]))) rescue Errno::ENOENT => e return_error "Could not load resource configuration", e end end if options[:datasetconfig] begin @dataset_config = JSON.parse(File.read((Pathname.new(options[:datasetconfig])).absolute? ? options[:datasetconfig] : File.join(@dir, options[:datasetconfig]))) rescue Errno::ENOENT => e return_error "Could not load dataset configuration", e end end if options[:configuration] begin @api_config = JSON.parse(File.read((Pathname.new(options[:configuration])).absolute? ? options[:configuration] : File.join(@dir, options[:configuration]))) rescue Errno::ENOENT => e return_error "Could not load configuration", e end end if !@api_config.nil? @ckan_client = CkanClient::Client.new(@api_config["ckan_api"]["url"], @api_config["ckan_api"]["api_key"]) @mail_server = @api_config["email_server"] @mail_receivers = @api_config["notification_receiver"] end @update_modified = options[:updatemodified] @overwrite = options[:overwrite] end
read_resource(source)
click to toggle source
# File lib/ckancli/commands/upload.rb, line 406 def read_resource(source) # check if URL or file unless source =~ /^http(s)?/ source = File.new( source ) end source end
read_schema(path)
click to toggle source
# File lib/ckancli/commands/upload.rb, line 415 def read_schema(path) schema = Csvlint::Schema.load_from_uri(path, false) schema end
return_error(message, exception = nil)
click to toggle source
# File lib/ckancli/commands/upload.rb, line 365 def return_error(message, exception = nil) show_error(message, exception) exit 1 end
show_error(message, exception = nil)
click to toggle source
# File lib/ckancli/commands/upload.rb, line 371 def show_error(message, exception = nil) if !exception.nil? message = "#{message} - #{exception.message}" end puts " #{message} ".white.on_red log_fatal message, exception end
show_info(message)
click to toggle source
# File lib/ckancli/commands/upload.rb, line 381 def show_info(message) puts " #{message} " log_info message end
task(message) { || ... }
click to toggle source
# File lib/ckancli/commands/upload.rb, line 387 def task(message) log_info message spinner = TTY::Spinner.new("[:spinner] #{message}...") spinner.auto_spin ok, message, exception = yield spinner.stop(ok ? "OK".green : "ERROR".red) if !message.nil? message = " #{message}" if !exception.nil? show_error message, exception else show_info message end end end
update_dataset()
click to toggle source
# File lib/ckancli/commands/upload.rb, line 341 def update_dataset() puts "" show_info "Starting dataset update..." task("Updating dataset") do ok = false msg = nil exception = nil @ckan_client.update_package(@dataset_config["result"]["id"], @dataset_config["result"].clone, true){ |response, status_ok| ok = status_ok if !ok msg = "HTTP error #{response.code}" log_error response.body end } [ok, msg, exception] end show_info "End of dataset update" end
upload()
click to toggle source
# File lib/ckancli/commands/upload.rb, line 285 def upload() puts "" show_info "Starting upload..." @resources.each do |res| next unless res[:valid] task("Uploading resource: #{File.basename(res[:path])}") do res[:log].info "Uploading resource..." ok = true msg = nil exception = nil params = @resource_config["result"].clone if @update_modified params["last_modified"] = JSON.parse(Time.new.to_json) end # check if resource exists and if overwriting if !params["id"].nil? && !@overwrite resource = @ckan_client.get_resource(params["id"]) if !resource.nil? ok = false msg = "Resource already exists - skipping. Use parameter 'overwrite' to overwrite changes." end end if ok @ckan_client.create_or_update_resource(params, File.new(res[:path], 'rb'), true){ |response, status_ok| ok = status_ok if !ok msg = "HTTP error #{response.code}" log_error response.body end } end if !ok res[:log].error msg res[:error] = msg else res[:log].info "OK - upload successfull" end res[:log].info "End of upload" [ok, msg, exception] end end show_info "End of upload" end
validate()
click to toggle source
# File lib/ckancli/commands/upload.rb, line 175 def validate() puts "" show_info "Starting validations..." @resources.each do |res| task("Validating resource: #{File.basename(res[:path])}") do ok = false msg = nil exception = nil begin res[:log].info "Validating resource..." res[:valid] = false source = read_resource(res[:path]) schema = read_schema(@schema) if schema.class == Csvlint::Schema && schema.description == "malformed" msg = "Invalid schema - malformed JSON" else validator = Csvlint::Validator.new(source, {}, schema) if validator.errors.length > 0 || validator.warnings.length > 0 msg = "#{validator.errors.length} errors, #{validator.warnings.length} warnings" res[:summary] = { :errors => validator.errors.map { |v| format_validation(v, schema) }, :warnings => validator.warnings.map { |v| format_validation(v, schema) } } res[:log].error res[:summary].to_json end ok = validator.valid? res[:valid] = ok end rescue Errno::ENOENT, OpenURI::HTTPError => e msg = "Could not load file - #{e.message}" rescue Csvlint::Csvw::MetadataError => e msg = "Invalid schema metadata - #{e.message}" rescue StandardError => e msg = "Could not validate resource" exception = e end if !exception.nil? res[:log].error msg, exception res[:error] = msg elsif !msg.nil? res[:log].info msg res[:error] = msg else res[:log].info "OK - Validation successfull" end res[:log].info "End of validation" [ok, msg, exception] end end show_info "End of validations" end