class MerryGoRound::Aggregator

Public Instance Methods

aggregate!() click to toggle source
# File lib/merry_go_round/aggregator.rb, line 5
def aggregate!
  from_redis
  compound
end

Private Instance Methods

compound() click to toggle source

Aggregate existing aggregations (i.e. hour -> day)

# File lib/merry_go_round/aggregator.rb, line 45
def compound
  # Get list of known keys
  keys = Aggregation.select('DISTINCT(key) AS key').collect(&:key)

  # Aggregate each key
  keys.each do |key|
    compound_key(key)
  end
end
compound_key(key) click to toggle source
# File lib/merry_go_round/aggregator.rb, line 55
def compound_key(key)
  now = Time.now.utc

  # Loop through all of the granularities except for the first one
  MerryGoRound.granularities.each_with_index do |granularity, index|
    # Skip minutes since the Redis aggregator does those
    next if index == 0

    # Get starting point
    start = start_timestamp(key, granularity).to_i

    # Time interval for granularity
    # TODO: Use `window` instead of `seconds`
    interval = seconds(granularity)

    # Get the previous granularity
    prev_gran = MerryGoRound.granularities[index - 1]

    # Loop through the results
    while start < now.to_i
      range = window(granularity, Time.at(start).utc)
      break unless range.first < now

      # Check time interval to see if we have enough data to do this granularity
      break if Aggregation.where('key = ? AND granularity = ? AND timestamp = ?', key, granularity, range.first).first

      children = Aggregation.where('key = ? AND granularity = ? AND timestamp >= ? AND timestamp <= ?', key, prev_gran, range.first, range.last).order('timestamp ASC')
      break unless children.length > 0

      # Get the aggregated value
      # TODO: This should probably done in SQL
      value = children.collect(&:value).reduce(:+)

      # Create the aggregation
      agg = Aggregation.create(key: key, value: value, granularity: granularity, timestamp: range.first)

      # Associate the children with their parent
      children.each do |child|
        child.parent_id = agg.id
        child.save
      end

      # Increment the timestamp
      start += interval
    end
  end
end
from_redis() click to toggle source

Aggregate stuff since the last aggregation in Redis

# File lib/merry_go_round/aggregator.rb, line 17
def from_redis
  # Aggregate to the first granularity
  redis_granularity = MerryGoRound.granularities.first

  # Get entry keys from Redis
  redis_keys = redis.keys 'entry-*'

  # Loop through the Redis keys
  redis_keys.each do |redis_key|
    # Extract the timestamp
    timestamp = Time.at(redis_key.sub('entry-', '').to_i).utc

    # TODO: Make sure this timestamp hasn't already been aggregated

    # Get the hash of Merry Go Round keys and values
    hash = redis.hgetall redis_key

    # Loop through each key/value in the hash
    hash.each do |key, value|
      Aggregation.create(key: key, value: value, timestamp: timestamp, granularity: redis_granularity)
    end

    # Delete the key
    redis.del redis_key
  end
end
redis() click to toggle source
# File lib/merry_go_round/aggregator.rb, line 12
def redis
  MerryGoRound.redis
end
start_timestamp(key, granularity) click to toggle source
# File lib/merry_go_round/aggregator.rb, line 103
def start_timestamp(key, granularity)
  # Search at this level
  result = Aggregation.select('MAX(timestamp) AS timestamp').where(key: key, granularity: granularity).order('timestamp DESC').first
  if result and result.timestamp
    # TODO: Possibily add 1
    return result.timestamp
  end

  # Go down a level
  prev_gran = MerryGoRound.granularities[MerryGoRound.granularities.index(granularity) - 1]
  result = Aggregation.select('MIN(timestamp) AS timestamp').where(key: key, granularity: prev_gran).order('timestamp ASC').first
  return result.timestamp if result and result.timestamp

  nil
end