class Bio::BaseSpace::MultipartUpload

Multipart file upload class.

TODO This file is not yet ported as the multipartFileUpload class is just mentioned in the comment section of the BaseSpaceAPI file.

Constants

StartTime
Status

Public Class Methods

new(api, a_id, local_file, file_object, cpu_count, part_size, temp_dir, verbose) click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 136
def initialize(api, a_id, local_file, file_object, cpu_count, part_size, temp_dir, verbose)
  @api            = api
  @analysis_id    = a_id
  @local_file     = local_file
  @remote_file    = file_object
  @part_size      = part_size
  @cpu_count      = cpu_count
  @verbose        = verbose
  @temp_dir       = temp_dir
  @status         = 'Initialized'
  @start_time     = -1
  #@repeat_count  = 0             # number of chunks we uploaded multiple times
  setup
end

Public Instance Methods

__cleanUp__() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 207
def __cleanUp__
  self.stats[0] += 1
end
finalize() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 239
def finalize
  raise Exception('Cannot finalize a transfer with running threads.') if self.getRunningThreadCount()
  if self.Status=='Running'
    # code here for
    self.Status=='Completed'
  else
    raise Exception('To finalize the status of the transfer must be "Running."')
  end
end
getFileResponse() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 268
def getFileResponse
  return self.remoteFile
end
getProgressRatio() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 295
def getProgressRatio
  currentQ = float(self.tasks.qsize() - len(self.consumers))
  # NOTE Python sublist [:6] already ported to Ruby [0..5]
  return str(float(self.totalTask - currentQ) / self.totalTask)[0..5]
end
getRunningThreadCount() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 272
def getRunningThreadCount
  return sum(self.consumers.map { |c| c.is_alive() })
end
getRunningTime() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 282
def getRunningTime
  if self.StartTime==-1
    return 0
  else
    return time.time() - self.StartTime
  end
end
getStatus() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 264
def getStatus
  return self.Status
end
getTotalTransfered() click to toggle source

Returns the total data amoun transfered in Gb

# File lib/basespace/model/multipart_upload.rb, line 291
def getTotalTransfered
  return float((self.totalTask - self.tasks.qsize())*self.partSize) / 1000.0
end
getTransRate() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 276
def getTransRate
  # tasks completed                        size of file-parts
  # NOTE Python sublist [:6] already ported to Ruby [0..5]
  return str((self.totalTask - self.tasks.qsize())*self.partSize/self.getRunningTime())[0..5] + ' mb/s'
end
haltUpload() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 259
def haltUpload
  self.consumers.each { |c| c.terminate() }
  self.Status = 'Terminated'
end
hasFinished() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 249
def hasFinished
  return 0 if self.Status == 'Initialized'
  return !(self.getRunningThreadCount() > 0)
end
pauseUpload() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 254
def pauseUpload
  self.pauseEvent.set()
  self.Status = 'Paused'
end
run() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 166
def run
  while @status == 'Paused' or __check_queue__
    time.sleep(self.wait)
  end
end
setup() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 172
def setup
  # determine the
  totalSize = os.path.getsize(self.localFile)
  fileCount = int(math.ceil(totalSize/(self.partSize*1024.0*1000)))
  
  if self.verbose 
    print "TotalSize " + str(totalSize)
    print "Using split size " + str(self.partSize) +"Mb"
    print "Filecount " + str(fileCount)
    print "CPUs " + str(self.cpuCount)
  end
  
  # Establish communication queues
  self.tasks = multiprocessing.JoinableQueue()
  self.completedPool = multiprocessing.Queue()
  [1..fileCount].each { |i|         # set up the task queue
    t = uploadTask(self.api,self.remoteFile.Id, i, fileCount, self.localFile, 0)
    self.tasks.put(t)
  }
  self.totalTask  = self.tasks.qsize()
  
  # create consumers
  self.pauseEvent = multiprocessing.Event()
  self.haltEvent = multiprocessing.Event()
  if self.verbose
    print 'Creating %d consumers' % self.cpuCount
    print "queue size " + str(self.tasks.qsize())
  end
  # NOTE Original code -- note the strange indent. Variables i and c not used. Buggy code?
  # self.consumers = [ Consumer(self.tasks, self.completedPool,self.pauseEvent,self.haltEvent) for i in xrange(self.cpuCount) ]
  #   for c in self.consumers: self.tasks.put(None)   # add poisson pill
  self.consumers = [0..self.cpuCount].map { |i| Consumer(self.tasks, self.completedPool, self.pauseEvent, self.haltEvent) }
  self.consumers.each { |c| self.tasks.put(nil) }
end
startUpload(returnOnFinish = 0, testInterval = 5) click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 211
def startUpload(returnOnFinish = 0, testInterval = 5)
  raise Exception('Cannot resume a ' + self.Status + ' multi-part upload session.') if self.Status == 'Terminated' or self.Status == 'Completed'
      
  if self.Status == 'Initialized'
    self.StartTime = time.time()
    self.consumers.each { |w|
      w.start()
    }
  elsif self.Status == 'Paused'
    self.pauseEvent.clear()
  end
  self.Status = 'Running'
      
  # If returnOnFinish is set
  if returnOnFinish
    i=0
    while not self.hasFinished()
      print str(i) + ': ' + str(self) if self.verbose and i
      time.sleep(testInterval)
      i+=1
    end
    self.finalize()
    return 1
  else
    return 1
  end
end
to_s() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 151
def to_s
  # TODO fix this.
  # NOTE Python sublist notation [:5] already changed to Ruby [0..4]
  return "MPU -  Stat: " + @status +  ", LiveThread: " + str(self.getRunningThreadCount()) + \
              ", RunTime: " + str(self.getRunningTime())[0..4] + 's' + \
              ", Q-size " + str(self.tasks.qsize()) + \
              ", Completed " + str(self.getProgressRatio()) + \
              ", AVG TransferRate " + self.getTransRate() + \
              ", Data transfered " + str(self.getTotalTransfered())[0..4] + 'Gb'
end
to_str() click to toggle source
# File lib/basespace/model/multipart_upload.rb, line 162
def to_str
  return self.inspect
end