class Asperalm::Cli::Plugins::Preview
Constants
- ACTIONS
- AK_MARKER_FILE
- DEFAULT_PREVIEWS_FOLDER
- LOCAL_STORAGE_PCVL
- PREVIEW_BASENAME
basename of preview files
- PREVIEW_FOLDER_SUFFIX
defined by node API: suffix for folder containing previews
- PREV_GEN_TAG
special tag to identify transfers related to generator
- TMP_DIR_PREFIX
subfolder in system tmp folder
Attributes
option_file_access[RW]
option_folder_reset_cache[RW]
option_overwrite[RW]
option_previews_folder[RW]
option_skip_format
has special accessors
option_skip_folders[RW]
Public Class Methods
new(env)
click to toggle source
Calls superclass method
Asperalm::Cli::BasicAuthPlugin::new
# File lib/asperalm/cli/plugins/preview.rb, line 34 def initialize(env) super(env) @skip_types=[] @default_transfer_spec=nil # by default generate all supported formats @preview_formats_to_generate=Asperalm::Preview::Generator::PREVIEW_FORMATS.clone # options for generation @gen_options=Asperalm::Preview::Options.new # link CLI options to gen_info attributes self.options.set_obj_attr(:skip_format,self,:option_skip_format,[]) # no skip self.options.set_obj_attr(:folder_reset_cache,self,:option_folder_reset_cache,:no) self.options.set_obj_attr(:skip_types,self,:option_skip_types) self.options.set_obj_attr(:previews_folder,self,:option_previews_folder,DEFAULT_PREVIEWS_FOLDER) self.options.set_obj_attr(:skip_folders,self,:option_skip_folders,[]) # no skip self.options.set_obj_attr(:overwrite,self,:option_overwrite,:mtime) self.options.set_obj_attr(:file_access,self,:option_file_access,:local) self.options.add_opt_list(:skip_format,Asperalm::Preview::Generator::PREVIEW_FORMATS,'skip this preview format (multiple possible)') self.options.add_opt_list(:folder_reset_cache,[:no,:header,:read],'force detection of generated preview by refresh cache') self.options.add_opt_simple(:skip_types,'skip types in comma separated list') self.options.add_opt_simple(:previews_folder,'preview folder in storage root') self.options.add_opt_simple(:temp_folder,'path to temp folder') self.options.add_opt_simple(:skip_folders,'list of folder to skip') self.options.add_opt_simple(:case,'test case name') self.options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file') self.options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository') self.options.set_option(:temp_folder,Dir.tmpdir) # add other options for generator (and set default values) Asperalm::Preview::Options::DESCRIPTIONS.each do |opt| self.options.set_obj_attr(opt[:name],@gen_options,opt[:name],opt[:default]) if opt.has_key?(:values) self.options.add_opt_list(opt[:name],opt[:values],opt[:description]) elsif [:yes,:no].include?(opt[:default]) self.options.add_opt_boolean(opt[:name],opt[:description]) else self.options.add_opt_simple(opt[:name],opt[:description]) end end self.options.parse_options! raise 'skip_folder shall be an Array, use @json:[...]' unless @option_skip_folders.is_a?(Array) @tmp_folder=File.join(self.options.get_option(:temp_folder,:mandatory),"#{TMP_DIR_PREFIX}.#{SecureRandom.uuid}") FileUtils.mkdir_p(@tmp_folder) Log.log.debug("tmpdir: #{@tmp_folder}") end
Public Instance Methods
do_transfer(direction,folder_id,source_filename,destination='/')
click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 166 def do_transfer(direction,folder_id,source_filename,destination='/') raise "error" if destination.nil? and direction.eql?('receive') if @default_transfer_spec.nil? # make a dummy call to get some default transfer parameters res=@api_node.create('files/upload_setup',{'transfer_requests'=>[{'transfer_request'=>{'paths'=>[{}],'destination_root'=>'/'}}]}) sample_transfer_spec=res[:data]['transfer_specs'].first['transfer_spec'] # get ports, anyway that should be 33001 for both. add remote_user ? @default_transfer_spec=['ssh_port','fasp_port'].inject({}){|h,e|h[e]=sample_transfer_spec[e];h} # note: we use the same address for ascp than for node api instead of the one from upload_setup @default_transfer_spec.merge!({ 'token' => "Basic #{Base64.strict_encode64("#{@access_key_self['id']}:#{self.options.get_option(:password,:mandatory)}")}", 'remote_host' => @transfer_server_address, 'remote_user' => Fasp::ACCESS_KEY_TRANSFER_USER }) end tspec=@default_transfer_spec.merge({ 'direction' => direction, 'paths' => [{'source'=>source_filename}], 'tags' => { 'aspera' => { PREV_GEN_TAG => true, 'node' => { 'access_key' => @access_key_self['id'], 'file_id' => folder_id }}} }) # force destination # tspec['destination_root']=destination self.transfer.option_transfer_spec_deep_merge({'destination_root'=>destination}) Main.result_transfer(self.transfer.start(tspec,{:src=>:node_gen4})) end
entry_preview_folder_name(entry)
click to toggle source
defined by node api
# File lib/asperalm/cli/plugins/preview.rb, line 231 def entry_preview_folder_name(entry) "#{entry['id']}#{PREVIEW_FOLDER_SUFFIX}" end
execute_action()
click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 348 def execute_action command=self.options.get_next_command(ACTIONS) unless [:check,:test].include?(command) # this will use node api @api_node=basic_auth_api @transfer_server_address=URI.parse(@api_node.params[:base_url]).host # get current access key @access_key_self=@api_node.read('access_keys/self')[:data] # TODO: check events is activated here: # note that docroot is good to look at as well node_info=@api_node.read('info')[:data] Log.log.debug("root: #{node_info['docroot']}") @access_remote=@option_file_access.eql?(:remote) Log.log.debug("remote: #{@access_remote}") Log.log.debug("access key info: #{@access_key_self}") #TODO: can the previews folder parameter be read from node api ? @option_skip_folders.push('/'+@option_previews_folder) if @access_remote # note the filter "name", it's why we take the first one @previews_folder_entry=get_folder_entries(@access_key_self['root_file_id'],{:name=>@option_previews_folder}).first raise CliError,"Folder #{@option_previews_folder} does not exist on node. Please create it in the storage root, or specify an alternate name." if @previews_folder_entry.nil? else raise "only local storage allowed in this mode" unless @access_key_self['storage']['type'].eql?('local') @local_storage_root=@access_key_self['storage']['path'] #TODO: option to override @local_storage_root='xxx' @local_storage_root=@local_storage_root[LOCAL_STORAGE_PCVL.length..-1] if @local_storage_root.start_with?(LOCAL_STORAGE_PCVL) #TODO: windows could have "C:" ? raise "not local storage: #{@local_storage_root}" unless @local_storage_root.start_with?('/') raise CliError,"Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root) @local_preview_folder=File.join(@local_storage_root,@option_previews_folder) raise CliError,"Folder #{@local_preview_folder} does not exist locally. Please create it, or specify an alternate name." unless File.directory?(@local_preview_folder) # protection to avoid clash of file id for two different access keys marker_file=File.join(@local_preview_folder,AK_MARKER_FILE) Log.log.debug("marker file: #{marker_file}") if File.exist?(marker_file) ak=File.read(marker_file) raise "mismatch access key in #{marker_file}: contains #{ak}, using #{@access_key_self['id']}" unless @access_key_self['id'].eql?(ak) else File.write(marker_file,@access_key_self['id']) end end end case command when :scan scan_folder_files({ 'id' => @access_key_self['root_file_id'], 'name' => '/', 'type' => 'folder', 'path' => '/' }) return Main.result_status('scan finished') when :events iteration_data=[] iteration_persistency=nil if self.options.get_option(:once_only,:mandatory) iteration_persistency=PersistencyFile.new( data: iteration_data, ids: ['preview_iteration_events',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)]) end iteration_data[0]=process_file_events(iteration_data[0]) iteration_persistency.save unless iteration_persistency.nil? return Main.result_status('events finished') when :trevents iteration_data=[] iteration_persistency=nil if self.options.get_option(:once_only,:mandatory) iteration_persistency=PersistencyFile.new( data: iteration_data, ids: ['preview_iteration_transfer',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)]) end iteration_data[0]=process_transfer_events(iteration_data[0]) iteration_persistency.save unless iteration_persistency.nil? return Main.result_status('trevents finished') when :folder file_id=self.options.get_next_argument('file id') file_info=@api_node.read("files/#{file_id}")[:data] scan_folder_files(file_info) return Main.result_status('file finished') when :check Asperalm::Preview::Utils.check_tools(@skip_types) return Main.result_status('tools validated') when :test format = self.options.get_next_argument('format',Asperalm::Preview::Generator::PREVIEW_FORMATS) source = self.options.get_next_argument('source file') dest=preview_filename(format,self.options.get_option(:case,:optional)) g=Asperalm::Preview::Generator.new(@gen_options,source,dest,@tmp_folder) raise "format not supported: #{format}" unless g.supported? g.generate return Main.result_status("generated: #{dest}") end ensure FileUtils.rm_rf(@tmp_folder) end
generate_preview(entry)
click to toggle source
generate preview files for one folder entry (file) if necessary entry must contain “parent_file_id” if remote.
# File lib/asperalm/cli/plugins/preview.rb, line 241 def generate_preview(entry) #Log.log.debug(">>>> #{entry}".red) # folder where previews will be generated for this particular entry local_entry_preview_dir=String.new # prepare generic information gen_infos=@preview_formats_to_generate.map do |preview_format| { :preview_format => preview_format, :base_dest => preview_filename(preview_format) } end # lets gather some infos on possibly existing previews # it depends if files access locally or remotely if @access_remote get_infos_remote(gen_infos,entry,local_entry_preview_dir) else # direct local file system access get_infos_local(gen_infos,entry,local_entry_preview_dir) end # here we have the status on preview files # let's find if they need generation gen_infos.select! do |gen_info| # if it exists, what about overwrite policy ? if gen_info[:preview_exist] case @option_overwrite when :always # continue: generate when :never # never overwrite next false when :mtime # skip if preview is newer than original next false if gen_info[:preview_newer_than_original] end end # need generator for further checks gen_info[:generator]=Asperalm::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'],false) # get conversion_type (if known) and check if supported next false unless gen_info[:generator].supported? # shall we skip it ? next false if @skip_types.include?(gen_info[:generator].conversion_type) # ok we need to generate true end return if gen_infos.empty? # create folder if needed FileUtils.mkdir_p(local_entry_preview_dir) if @access_remote raise 'missing parent_file_id in entry' if entry['parent_file_id'].nil? # download original file to temp folder do_transfer('receive',entry['parent_file_id'],entry['name'],@tmp_folder) end Log.log.info("source: #{entry['id']}: #{entry['path']})") gen_infos.each do |gen_info| gen_info[:generator].generate rescue nil end if @access_remote # upload do_transfer('send',@previews_folder_entry['id'],local_entry_preview_dir) # cleanup after upload FileUtils.rm_rf(local_entry_preview_dir) File.delete(File.join(@tmp_folder,entry['name'])) end # force read file updated previews if @option_folder_reset_cache.eql?(:read) @api_node.read("files/#{entry['id']}") end rescue => e Log.log.error("#{e.message}") Log.log.debug(e.backtrace.join("\n").red) end
get_folder_entries(file_id,request_args=nil)
click to toggle source
/files/id/files is normally cached in redis, but we can discard the cache but /files/id is not cached
# File lib/asperalm/cli/plugins/preview.rb, line 105 def get_folder_entries(file_id,request_args=nil) headers={'Accept'=>'application/json'} headers.merge!({'X-Aspera-Cache-Control'=>'no-cache'}) if @option_folder_reset_cache.eql?(:header) return @api_node.call({:operation=>'GET',:subpath=>"files/#{file_id}/files",:headers=>headers,:url_params=>request_args})[:data] #return @api_node.read("files/#{file_id}/files",request_args)[:data] end
get_infos_local(gen_infos,entry,local_entry_preview_dir)
click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 196 def get_infos_local(gen_infos,entry,local_entry_preview_dir) local_original_filepath=File.join(@local_storage_root,entry['path']) original_mtime=File.mtime(local_original_filepath) # out local_entry_preview_dir.replace(File.join(@local_preview_folder, entry_preview_folder_name(entry))) gen_infos.each do |gen_info| gen_info[:src]=local_original_filepath gen_info[:dst]=File.join(local_entry_preview_dir, gen_info[:base_dest]) gen_info[:preview_exist]=File.exist?(gen_info[:dst]) gen_info[:preview_newer_than_original] = (gen_info[:preview_exist] and (File.mtime(gen_info[:dst])>original_mtime)) end end
get_infos_remote(gen_infos,entry,local_entry_preview_dir)
click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 209 def get_infos_remote(gen_infos,entry,local_entry_preview_dir) #Log.log.debug(">>>> get_infos_remote #{entry}".red) # store source directly here local_original_filepath=File.join(@tmp_folder,entry['name']) #original_mtime=DateTime.parse(entry['modified_time']) # out: where previews are generated local_entry_preview_dir.replace(File.join(@tmp_folder,entry_preview_folder_name(entry))) file_info=@api_node.read("files/#{entry['id']}")[:data] #TODO: this does not work because previews is hidden in api (gen4) #this_preview_folder_entries=get_folder_entries(@previews_folder_entry['id'],{:name=>@entry_preview_folder_name}) # TODO: use gen3 api to list files and get date gen_infos.each do |gen_info| gen_info[:src]=local_original_filepath gen_info[:dst]=File.join(local_entry_preview_dir, gen_info[:base_dest]) # TODO: use this_preview_folder_entries (but it's hidden) gen_info[:preview_exist]=file_info.has_key?('preview') # TODO: get change time and compare, useful ? gen_info[:preview_newer_than_original] = gen_info[:preview_exist] end end
option_skip_format()
click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 97 def option_skip_format return @preview_formats_to_generate.map{|i|i.to_s}.join(',') end
option_skip_format=(value)
click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 93 def option_skip_format=(value) @preview_formats_to_generate.delete(value) end
option_skip_types()
click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 89 def option_skip_types return @skip_types.map{|i|i.to_s}.join(',') end
option_skip_types=(value)
click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 80 def option_skip_types=(value) @skip_types=[] value.split(',').each do |v| s=v.to_sym raise "not supported: #{v}" unless Asperalm::Preview::FileTypes::CONVERSION_TYPES.include?(s) @skip_types.push(s) end end
preview_filename(preview_format,filename=PREVIEW_BASENAME)
click to toggle source
# File lib/asperalm/cli/plugins/preview.rb, line 235 def preview_filename(preview_format,filename=PREVIEW_BASENAME) return "#{filename}.#{preview_format.to_s}" end
process_file_events(iteration_token)
click to toggle source
requests recent events on node api and process newly modified folders
# File lib/asperalm/cli/plugins/preview.rb, line 138 def process_file_events(iteration_token) # get new file creation by access key (TODO: what if file already existed?) events_filter={ 'access_key'=>@access_key_self['id'], 'type'=>'file.*' } # and optionally by iteration token events_filter['iteration_token']=iteration_token unless iteration_token.nil? events=@api_node.read('events',events_filter)[:data] return if events.empty? events.each do |event| # process only files next unless event.dig('data','type').eql?('file') file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil next if file_entry.nil? next unless @option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty? file_entry['parent_file_id']=event['data']['parent_file_id'] if event['types'].include?('file.deleted') Log.log.error('TODO'.red) end if event['types'].include?('file.deleted') generate_preview(file_entry) end end # write new iteration file return events.last['id'].to_s end
process_transfer_events(iteration_token)
click to toggle source
old version based on folders
# File lib/asperalm/cli/plugins/preview.rb, line 113 def process_transfer_events(iteration_token) events_filter={ 'access_key'=>@access_key_self['id'], 'type'=>'download.ended' } # optionally by iteration token events_filter['iteration_token']=iteration_token unless iteration_token.nil? events=@api_node.read('events',events_filter)[:data] return if events.empty? events.each do |event| next unless event['data']['direction'].eql?('receive') next unless event['data']['status'].eql?('completed') next unless event['data']['error_code'].eql?(0) next unless event['data'].dig('tags','aspera',PREV_GEN_TAG).nil? folder_id=event.dig('data','tags','aspera','node','file_id') folder_id||=event.dig('data','file_id') next if folder_id.nil? folder_entry=@api_node.read("files/#{folder_id}")[:data] rescue nil next if folder_entry.nil? scan_folder_files(folder_entry) end return events.last['id'].to_s end
scan_folder_files(top_entry)
click to toggle source
scan all files in provided folder entry
# File lib/asperalm/cli/plugins/preview.rb, line 313 def scan_folder_files(top_entry) Log.log.debug("scan: #{top_entry}") # don't use recursive call, use list instead items_to_process=[top_entry] while !items_to_process.empty? entry=items_to_process.shift Log.log.debug("item:#{entry}") case entry['type'] when 'file' generate_preview(entry) when 'link' Log.log.debug('Ignoring link.') when 'folder' if @option_skip_folders.include?(entry['path']) Log.log.debug("#{entry['path']} folder (skip)".bg_red) else Log.log.debug("#{entry['path']} folder") # get folder content folder_entries=get_folder_entries(entry['id']) # process all items in current folder folder_entries.each do |folder_entry| # add path for older versions of ES if !folder_entry.has_key?('path') folder_entry['path']=(entry['path'].eql?('/')?'':entry['path'])+'/'+folder_entry['name'] end folder_entry['parent_file_id']=entry['id'] items_to_process.push(folder_entry) end end else Log.log.warn("unknown entry type: #{entry['type']}") end end end