class Jober::ARLoop

Public Class Methods

batch_size(bs) click to toggle source
# File lib/jober/ar_loop.rb, line 3
def batch_size(bs)
  @batch_size = bs
end
get_batch_size() click to toggle source
# File lib/jober/ar_loop.rb, line 7
def get_batch_size
  @batch_size ||= 1000
end

Public Instance Methods

log_prefix() click to toggle source
# File lib/jober/ar_loop.rb, line 67
def log_prefix
  ''
end
proxy() click to toggle source
# File lib/jober/ar_loop.rb, line 12
def proxy
  raise "implement me, should return AR.proxy, ex: User.where('years > 18')"
end
reset_last_batch_id() click to toggle source
# File lib/jober/ar_loop.rb, line 75
def reset_last_batch_id
  Jober.redis.del(store_key('lastbatch'))
end
run() click to toggle source
# File lib/jober/ar_loop.rb, line 16
def run
  prox = proxy

  prefix = log_prefix

  if @worker_id && @workers_count && @workers_count > 1 && !@opts[:no_auto_proxy]
    cond = "id % #{@workers_count} = #{@worker_id}"
    prox = prox.where(cond)
    prefix += "(#{cond}) "
    info { "sharding enabled '#{cond}'" }
  end

  last_batch_id = if (_last_batch_id = get_store("lastbatch")) && !@opts[:no_last_batch]
    info { "Finded last_batch_id #{_last_batch_id} so start with it!" }
    prefix += "(#{_last_batch_id}:) "
    _last_batch_id
  end

  if @opts[:where]
    prox = prox.where(@opts[:where])
    prefix += "(cwhere) "
  end

  cnt = 0
  count = last_batch_id ? prox.where("id > ?", last_batch_id).count : prox.count
  info { "#{prefix}full count to process #{count}" }

  h = { :batch_size => self.class.get_batch_size }
  h[:start] = last_batch_id + 1 if last_batch_id

  t1 = Time.now
  prox.find_in_batches(h) do |batch|
    t = Time.now

    res = perform(batch)
    cnt += batch.size

    select_time = t - t1
    t1 = Time.now
    process_time = t1 - t

    info { "#{prefix}process batch #{res.inspect} | #{cnt} from #{count} | lastid: #{batch.last.id} (#{format("%.1f", select_time)}s, #{format("%.1f", process_time)}s)" }
    set_store("lastbatch", batch.last.id)
    break if stopped
  end

  reset_last_batch_id unless stopped

  info { "#{prefix}processed total #{cnt} #{summary_info}" }
end
summary_info() click to toggle source
# File lib/jober/ar_loop.rb, line 71
def summary_info
  nil
end