class BuildCache::DiskCache

Attributes

check_size_percent[RW]

Percent of time to check the cache size

dir[R]

The directory containing cached files

evict_percent[RW]

The percent of entries to evict (delete) if size is exceeded

logger[RW]

logger to use, if logger is not set, then messages will not be logged

max_cache_size[RW]

The maximum number of entries in the cache. Note that since we don't check the cache size every time, the actual size might exceed this number

permissions[R]

The Linux permissions that cached files should have

Public Class Methods

new(dir='/tmp/cache', permissions=0666) click to toggle source
# File lib/buildcache.rb, line 41
def initialize dir='/tmp/cache', permissions=0666
  # Make sure 'dir' is not a file
  if (File.exist?(dir) && !File.directory?(dir))
    raise "DiskCache dir #{dir} should be a directory."
  end
  @dir = dir
  @permissions = permissions
  @enable_logging = false
  @check_size_percent = 20
  @max_cache_size = 5000
  @evict_percent = 20.0
  mkdir
end

Public Instance Methods

cache(input_files, metadata, dest_dir) { || ... } click to toggle source
# File lib/buildcache.rb, line 159
def cache input_files, metadata, dest_dir
  # Create the cache keys
  first_key = BuildCache.key_gen input_files, metadata
  second_key = metadata.to_s

  # If cache hit, copy the files to the dest_dir
  if (hit?first_key, second_key)
    begin
      cache_dir = get first_key, second_key
      log "cache hit #{cache_dir}"
      mkdir dest_dir
      FileUtils.cp_r(cache_dir + '/.', dest_dir)
      return Dir[cache_dir + '/*'].map { |pathname| File.basename pathname }
    rescue => e
      # Since we don't return, error counts as a cache miss
      log "ERROR: Could not retrieve cache entry contents. #{e.to_s}"
    end
  end

  # If cache miss, run the block and put the results in the cache
  files = yield
  output_files = files.map { |filename| File.join(dest_dir, filename) }
  # Check the cache again in case someone else populated it already
  unless (hit?first_key, second_key)
    cache_dir = set(first_key, second_key, output_files)
    log "cache miss, caching results to #{cache_dir}"
  end
  return files
end
check_cache_size() click to toggle source
# File lib/buildcache.rb, line 189
def check_cache_size
  log "checking cache size"
  entries = Dir[@dir + '/*/*']
  if entries.length > max_cache_size
    # If cache is locked for maintainance (lock exists and less than 8 hours old), then we skip the check
    lock_filename = File.join(@dir, 'cache_maintenance')
    return if (File.exist?(lock_filename) && (File.mtime(lock_filename) > Time.now - (8 * 60 * 60)))
    FileUtils.touch(lock_filename)

    log "evicting old cache entries"
    # evict some entries
    entries = entries.sort do |a,b|
      # evict entries that don't have a last_used file
      a_file = File.join(a, 'last_used')
      next -1 if !File.exist?a_file
      b_file = File.join(b, 'last_used')
      next 1 if !File.exist?b_file
      next File.mtime(a_file) <=> File.mtime(b_file)
    end

    entries_to_delete = (entries.length * evict_percent / 100).ceil
    entries[0..(entries_to_delete-1)].each { |entry| FileUtils.rm_rf(entry) }
    # Delete empty directories
    Dir[@dir + '/*'].each { |d| Dir.rmdir d if (File.directory?(d) && (Dir.entries(d) - %w[ . .. ]).empty?) }

    # Delete lock file
    FileUtils.rm lock_filename, :force => true
  end
end
get(first_key, second_key='') click to toggle source

Get the cache directory containing the contents corresponding to the keys

# File lib/buildcache.rb, line 127
def get first_key, second_key=''
  # TODO: validate inputs

  begin
    cache_dirs = Dir[File.join(@dir, first_key + '/*')]
    cache_dirs.each do |cache_dir|
      second_key_filename = cache_dir + '/second_key'
      # If second key filename is bad, we skip this directory
      if (!File.exist?(second_key_filename) || File.directory?(second_key_filename))
        next
      end
      second_key_file = File.open(second_key_filename, "r" )
      second_key_file.flock(File::LOCK_SH)
      out = second_key_file.read
      if (second_key.to_s == out)
        FileUtils.touch cache_dir + '/last_used'
        cache_dir = File.join(cache_dir, 'content')
        second_key_file.close
        return cache_dir if File.directory?(cache_dir)
      end
      second_key_file.close
    end
  rescue => e
    log "ERROR: Could not get cache entry. #{e.to_s}"
  end
  return nil
end
hit?(first_key, second_key='') click to toggle source
# File lib/buildcache.rb, line 155
def hit? first_key, second_key=''
  return get(first_key, second_key) != nil
end
set(first_key, second_key='', files=[]) click to toggle source
# File lib/buildcache.rb, line 55
def set first_key, second_key='', files=[]
  # TODO: validate inputs

  # If cache exists already, overwrite it.
  content_dir = get first_key, second_key
  second_key_file = nil

  begin
    if (content_dir.nil?)

      # Check the size of cache, and evict entries if too large
      check_cache_size if (rand(100) < check_size_percent)

      # Make sure cache dir doesn't exist already
      first_cache_dir = File.join(dir, first_key)
      if (File.exist?first_cache_dir)
        raise "BuildCache directory #{first_cache_dir} should be a directory" unless File.directory?(first_cache_dir)
      else
        FileUtils.mkpath(first_cache_dir)
      end
      num_second_dirs = Dir[first_cache_dir + '/*'].length
      cache_dir = File.join(first_cache_dir, num_second_dirs.to_s)
      # If cache directory already exists, then a directory must have been evicted here, so we pick another name
      while File.directory?cache_dir
        cache_dir = File.join(first_cache_dir, rand(num_second_dirs).to_s)
      end
      content_dir = File.join(cache_dir, '/content')
      FileUtils.mkpath(content_dir)

      # Create 'last_used' file
      last_used_filename = File.join(cache_dir, 'last_used')
      FileUtils.touch last_used_filename
      FileUtils.chmod(permissions, last_used_filename)

      # Copy second key
      second_key_file = File.open(cache_dir + '/second_key', 'w+')
      second_key_file.flock(File::LOCK_EX)
      second_key_file.write(second_key)

    else
      log "overwriting cache #{content_dir}"

      FileUtils.touch content_dir + '/../last_used'
      second_key_file = File.open(content_dir + '/../second_key', 'r')
      second_key_file.flock(File::LOCK_EX)
      # Clear any existing files out of cache directory
      FileUtils.rm_rf(content_dir + '/.')
    end

    # Copy files into content_dir
    files.each do |filename|
      FileUtils.cp(filename, content_dir)
    end
    FileUtils.chmod(permissions, Dir[content_dir + '/*'])

    # Release the lock
    second_key_file.close
    return content_dir
  rescue => e
    # Something went wrong, like a full disk or some other error.
    # Delete any work so we don't leave cache in corrupted state
    unless content_dir.nil?
      # Delete parent of content directory
      FileUtils.rm_rf(File.expand_path('..', content_dir))
    end
    log "ERROR: Could not set cache entry. #{e.to_s}"
    return 'ERROR: !NOT CACHED!'
  end
  
end

Private Instance Methods

log(message) click to toggle source
# File lib/buildcache.rb, line 225
def log message
  unless (@logger.nil?)
    @logger.info { "[BuildCache::DiskCache] #{message.to_s}" }
  end
end
mkdir(dir=@dir) click to toggle source
# File lib/buildcache.rb, line 221
def mkdir dir=@dir
  FileUtils.mkpath(dir) unless File.directory?(dir)
end