class Elasticrawl::Config

Represents the current configuration which is persisted to ~/.elasticrawl/ and contains 3 configuration files.

aws.yml - AWS access credentials unless stored in the environment

variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.

cluster.yml - Elastic MapReduce cluster config including instance groups. jobs.yml - Elastic MapReduce jobs config and the S3 bucket used for

storing data and logs.

This directory also contains the Elasticrawl SQLite database.

Constants

CONFIG_DIR
DATABASE_FILE
TEMPLATES_DIR
TEMPLATE_FILES

Attributes

access_key_id[R]
secret_access_key[R]

Public Class Methods

new(access_key_id = nil, secret_access_key = nil) click to toggle source

Sets the AWS access credentials needed for the S3 and EMR API calls.

# File lib/elasticrawl/config.rb, line 22
def initialize(access_key_id = nil, secret_access_key = nil)
  # Credentials have been provided to the init command.
  @access_key_id = access_key_id
  @secret_access_key = secret_access_key

  # If credentials are not set then check if they are available in aws.yml.
  if dir_exists?
    config = load_config('aws')
    key = config['access_key_id']
    secret = config['secret_access_key']

    @access_key_id ||= key unless key == 'ACCESS_KEY_ID'
    @secret_access_key ||= secret unless secret == 'SECRET_ACCESS_KEY'
  end

  # If credentials are still not set then check AWS environment variables.
  @access_key_id ||= ENV['AWS_ACCESS_KEY_ID']
  @secret_access_key ||= ENV['AWS_SECRET_ACCESS_KEY']

  # Set AWS credentials for use when accessing the S3 API.
  AWS.config(:access_key_id => @access_key_id,
             :secret_access_key => @secret_access_key)
end

Public Instance Methods

access_key_prompt() click to toggle source

Displayed by init command.

# File lib/elasticrawl/config.rb, line 139
def access_key_prompt
  prompt = "Enter AWS Access Key ID:"
  prompt += " [#{@access_key_id}]" if @access_key_id.present?

  prompt
end
bucket_exists?(bucket_name) click to toggle source

Checks if a S3 bucket name is in use.

# File lib/elasticrawl/config.rb, line 96
def bucket_exists?(bucket_name)
  begin
    s3 = AWS::S3.new
    s3.buckets[bucket_name].exists?

  rescue AWS::S3::Errors::SignatureDoesNotMatch => e
    raise AWSCredentialsInvalidError, 'AWS access credentials are invalid'
  rescue AWS::Errors::Base => s3e
    raise S3AccessError.new(s3e.http_response), s3e.message
  end
end
config_dir() click to toggle source

Returns the location of the config directory.

# File lib/elasticrawl/config.rb, line 47
def config_dir
  File.join(Dir.home, CONFIG_DIR)
end
create(bucket_name) click to toggle source

Creates the S3 bucket and config directory. Deploys the config templates and creates the sqlite database.

# File lib/elasticrawl/config.rb, line 110
def create(bucket_name)
  create_bucket(bucket_name)
  deploy_templates(bucket_name)
  load_database

  status_message(bucket_name, 'created')
end
delete() click to toggle source

Deletes the S3 bucket and config directory.

# File lib/elasticrawl/config.rb, line 119
def delete
  bucket_name = load_config('jobs')['s3_bucket_name']
  delete_bucket(bucket_name)
  delete_config_dir
  
  status_message(bucket_name, 'deleted')
end
delete_warning() click to toggle source

Displayed by destroy command to confirm deletion.

# File lib/elasticrawl/config.rb, line 128
def delete_warning
  bucket_name = load_config('jobs')['s3_bucket_name']

  message = ['WARNING:']
  message << "Bucket s3://#{bucket_name} and its data will be deleted"
  message << "Config dir #{config_dir} will be deleted"

  message.join("\n")
end
dir_exists?() click to toggle source

Checks if the configuration directory exists.

# File lib/elasticrawl/config.rb, line 52
def dir_exists?
  Dir.exists?(config_dir)
end
load_config(config_file) click to toggle source

Loads a YAML configuration file.

# File lib/elasticrawl/config.rb, line 57
def load_config(config_file)
  if dir_exists?
    begin
      config_file = File.join(config_dir, "#{config_file}.yml")
      config = YAML::load(File.open(config_file))

    rescue StandardError => e
      raise FileAccessError, e.message
    end
  else
    raise ConfigDirMissingError, 'Config dir missing. Run init command'
  end
end
load_database() click to toggle source

Loads the sqlite database. If no database exists it will be created and the database migrations will be run.

# File lib/elasticrawl/config.rb, line 73
def load_database
  if dir_exists?
    config = {
      'adapter' => 'sqlite3',
      'database' => File.join(config_dir, DATABASE_FILE),
      'pool' => 5,
      'timeout' => 5000
    }

    begin
      ActiveRecord::Base.establish_connection(config)
      ActiveRecord::Migrator.migrate(File.join(File.dirname(__FILE__), \
        '../../db/migrate'), ENV['VERSION'] ? ENV['VERSION'].to_i : nil )

    rescue StandardError => e
      raise DatabaseAccessError, e.message
    end
  else
    raise ConfigDirMissingError, 'Config dir missing. Run init command'
  end
end
secret_key_prompt() click to toggle source

Displayed by init command.

# File lib/elasticrawl/config.rb, line 147
def secret_key_prompt
  prompt = "Enter AWS Secret Access Key:"
  prompt += " [#{@secret_access_key}]" if @secret_access_key.present?

  prompt
end

Private Instance Methods

create_bucket(bucket_name) click to toggle source

Creates a bucket using the S3 API.

# File lib/elasticrawl/config.rb, line 156
def create_bucket(bucket_name)
  begin
    s3 = AWS::S3.new
    s3.buckets.create(bucket_name)

  rescue AWS::Errors::Base => s3e
    raise S3AccessError.new(s3e.http_response), s3e.message
  end
end
delete_bucket(bucket_name) click to toggle source

Deletes a bucket and its contents using the S3 API.

# File lib/elasticrawl/config.rb, line 167
def delete_bucket(bucket_name)
  begin
    s3 = AWS::S3.new
    bucket = s3.buckets[bucket_name]
    bucket.delete!

  rescue AWS::Errors::Base => s3e
    raise S3AccessError.new(s3e.http_response), s3e.message
  end
end
delete_config_dir() click to toggle source

Deletes the config directory including its contents.

# File lib/elasticrawl/config.rb, line 222
def delete_config_dir
  begin
    FileUtils.rm_r(config_dir) if dir_exists?

  rescue StandardError => e
    raise FileAccessError, e.message
  end
end
deploy_templates(bucket_name) click to toggle source

Creates config directory and copies config templates into it. Saves S3 bucket name to jobs.yml and AWS credentials to aws.yml.

# File lib/elasticrawl/config.rb, line 180
def deploy_templates(bucket_name)
  begin
    Dir.mkdir(config_dir, 0755) if dir_exists? == false

    TEMPLATE_FILES.each do |template_file|
      FileUtils.cp(File.join(File.dirname(__FILE__), TEMPLATES_DIR, template_file),
                   File.join(config_dir, template_file))
    end

    save_config('jobs', { 'BUCKET_NAME' => bucket_name })
    save_aws_config

  rescue StandardError => e
    raise FileAccessError, e.message
  end
end
save_aws_config() click to toggle source

Saves AWS access credentials to aws.yml unless they are configured as environment variables.

# File lib/elasticrawl/config.rb, line 199
def save_aws_config
  env_key = ENV['AWS_ACCESS_KEY_ID']
  env_secret = ENV['AWS_SECRET_ACCESS_KEY']

  creds = {}
  creds['ACCESS_KEY_ID'] = @access_key_id unless @access_key_id == env_key
  creds['SECRET_ACCESS_KEY'] = @secret_access_key \
    unless @secret_access_key == env_secret

  save_config('aws', creds)
end
save_config(template, params) click to toggle source

Saves config values by overwriting placeholder values in template.

# File lib/elasticrawl/config.rb, line 212
def save_config(template, params)
  config_file = File.join(config_dir, "#{template}.yml")
  config = File.read(config_file)

  params.map { |key, value| config = config.gsub(key, value) }

  File.open(config_file, 'w') { |file| file.write(config) }
end
status_message(bucket_name, state) click to toggle source

Notifies user of results of init or destroy commands.

# File lib/elasticrawl/config.rb, line 232
def status_message(bucket_name, state)
  message = ['', "Bucket s3://#{bucket_name} #{state}"]
  message << "Config dir #{config_dir} #{state}"

  state = 'complete' if state == 'created'
  message << "Config #{state}"

  message.join("\n")
end