class OpenStudio::Aws::Aws

Attributes

default_amis[R]
os_aws[R]

Deprecate OS_AWS object

save_directory[R]

Public Class Methods

new(options = {}) click to toggle source

default constructor to create the AWS class that can spin up server and worker instances. options are optional with the following support:

credentials => {:access_key_id, :secret_access_key, :region, :ssl_verify_peer}
proxy => {:host => "192.168.0.1", :port => "8808", :username => "user", :password => "password"}}

@param options [Hash] @option options [Boolean] :openstudio_version Version of OpenStudio in which to do the lookup for the server

AMIs. This cannot be used in conjunction with the :openstudio_server_version.

@option options [Boolean] :openstudio_server_version Version of OpenStudio Server in which to do the lookup for

the server AMIs. This cannot be used in conjunction with the :openstudio_server_version

@option options [Boolean] :stable (false) Find a stable version less than or equal to the version that is passed in the

version field
# File lib/openstudio/aws/aws.rb, line 64
def initialize(options = {})
  invalid_options = options.keys - VALID_OPTIONS
  if invalid_options.any?
    raise ArgumentError, "invalid option(s): #{invalid_options.join(', ')}"
  end

  # merge in some defaults
  defaults = {
    ami_lookup_version: 1,
    region: 'us-east-1',
    ssl_verify_peer: false,
    save_directory: '.'
  }
  options = defaults.merge(options)
  logger.info "AWS initialized with the options: #{options.except(:credentials)}"

  # set the save path
  @save_directory = File.expand_path options[:save_directory]
  FileUtils.mkdir_p @save_directory unless Dir.exist? @save_directory

  # read in the config.yml file to get the secret/private key
  if !options[:credentials]
    config_file = OpenStudio::Aws::Config.new

    # populate the credentials
    options[:credentials] =
      {
        access_key_id: config_file.access_key,
        secret_access_key: config_file.secret_key,
        region: options[:region],
        ssl_verify_peer: options[:ssl_verify_peer]
      }
  else
    options[:credentials][:region] = options[:region]
    options[:credentials][:ssl_verify_peer] = options[:ssl_verify_peer]
  end

  if options[:proxy]
    proxy_uri = nil
    if options[:proxy][:username]
      proxy_uri = "https://#{options[:proxy][:username]}:#{options[:proxy][:password]}@#{options[:proxy][:host]}:#{options[:proxy][:port]}"
    else
      proxy_uri = "https://#{options[:proxy][:host]}:#{options[:proxy][:port]}"
    end
    options[:proxy_uri] = proxy_uri
  end

  @os_aws = OpenStudioAwsWrapper.new(options)
  @os_cloudwatch = OpenStudioCloudWatch.new(options)

  @dockerized = options[:ami_lookup_version] == 3

  # this will grab the default version of openstudio ami versions
  # get the arugments for the AMI lookup
  ami_options = {}
  ami_options[:openstudio_server_version] = options[:openstudio_server_version] if options[:openstudio_server_version]
  ami_options[:openstudio_version] = options[:openstudio_version] if options[:openstudio_version]
  ami_options[:host] = options[:host] if options[:host]
  ami_options[:url] = options[:url] if options[:url]
  ami_options[:stable] = options[:stable] if options[:stable]

  @default_amis = OpenStudioAmis.new(options[:ami_lookup_version], ami_options).get_amis
end

Public Instance Methods

cluster_info() click to toggle source

Return information on the cluster instances as a hash. This includes IP addresses, host names, number of processors, etc. @return [Hash] Data about the configured cluster

# File lib/openstudio/aws/aws.rb, line 287
def cluster_info
  @os_aws.to_os_hash
end
create_server(options = {}) click to toggle source

command line call to create a new instance. This should be more tightly integrated with the os-aws.rb gem

# File lib/openstudio/aws/aws.rb, line 135
def create_server(options = {})
  defaults = {
    instance_type: 'm3.xlarge',
    security_groups: [],
    image_id: @default_amis[:server],
    user_id: 'unknown_user',

    # optional -- will default later
    associate_public_ip_address: nil,
    subnet_id: nil,
    ebs_volume_id: nil,
    aws_key_pair_name: nil,
    private_key_file_name: nil, # required if using an existing "aws_key_pair_name"
    tags: [],
    vpc_id: nil
  }
  options = defaults.merge(options)

  # for backwards compatibilty, still allow security_group
  if options[:security_group]
    warn 'Pass security_groups as an array instead of security_group. security_group will be deprecated in 0.4.0'
    options[:security_groups] = [options[:security_group]]
  end

  if options[:aws_key_pair_name]
    raise 'Must pass in the private_key_file_name' unless options[:private_key_file_name]
    raise "Private key was not found: #{options[:private_key_file_name]}" unless File.exist? options[:private_key_file_name]
  end

  if options[:security_groups].empty?
    # if the user has not specified any security groups, then create one called: 'openstudio-server-sg-v2'
    @os_aws.create_or_retrieve_default_security_group(tmp_name = 'openstudio-server-sg-v2.2',
                                                      vpc_id = options[:vpc_id])
  else
    @os_aws.security_groups = options[:security_groups]
  end

  @os_aws.create_or_retrieve_key_pair options[:aws_key_pair_name]

  # If using an already_existing key_pair, then you must pass in the private key file name
  if options[:aws_key_pair_name]
    @os_aws.load_private_key options[:private_key_file_name]
    @os_aws.private_key_file_name = options[:private_key_file_name]
  else
    # Save the private key if you did not pass in an already existing key_pair_name
    @os_aws.save_private_key @save_directory
  end

  user_data_file = @dockerized ? 'server_script.sh.docker.template' : 'server_script.sh.template'

  server_options = {
    user_id: options[:user_id],
    tags: options[:tags],
    subnet_id: options[:subnet_id],
    associate_public_ip_address: options[:associate_public_ip_address],
    user_data_file: user_data_file
  }

  server_options[:availability_zone] = options[:availability_zone] if options[:availability_zone]

  # save the worker pem and public to the directory
  # presently, this will always overwrite the worker key, is that okay? Is this really needed later?
  @os_aws.save_worker_keys @save_directory

  @os_aws.launch_server(options[:image_id], options[:instance_type], server_options)
end
create_workers(number_of_instances, options = {}, user_id = 'unknown_user') click to toggle source

create workers after the server has been created.

@param number_of_instances [Integer] Number of worker instances to create @param options [Hash] @option options [String] :instance_type Type of server to start (e.g. m3.medium, m3.xlarge, etc.)

# File lib/openstudio/aws/aws.rb, line 207
def create_workers(number_of_instances, options = {}, user_id = 'unknown_user')
  defaults = {
    instance_type: 'm2.4xlarge',
    security_groups: [],
    image_id: nil, # don't prescribe the image id so that it can determine later
    user_id: user_id,

    # optional -- will default later
    associate_public_ip_address: nil,
    subnet_id: nil,
    ebs_volume_id: nil,
    aws_key_pair_name: nil,
    private_key_file_name: nil, # required if using an existing "aws_key_pair_name",
    tags: []
  }
  options = defaults.merge(options)

  # for backwards compatibility, still allow security_group
  if options[:security_group]
    warn 'Pass security_groups as an array instead of security_group. security_group will be deprecated in 0.4.0'
    options[:security_groups] = [options[:security_group]]
  end

  # Get the right worker AMI ids based on the type of instance
  if options[:image_id].nil?
    options[:image_id] = determine_image_type(options[:instance_type])
  end

  raise "Can't create workers without a server instance running" if @os_aws.server.nil?

  user_data_file = @dockerized ? 'worker_script.sh.docker.template' : 'worker_script.sh.template'

  unless number_of_instances == 0
    worker_options = {
      user_id: options[:user_id],
      tags: options[:tags],
      subnet_id: options[:subnet_id],
      associate_public_ip_address: options[:associate_public_ip_address],
      user_data_file: user_data_file
    }

    # if options[:ebs_volume_size]
    #   worker_options[:ebs_volume_size] = options[:ebs_volume_size]
    # end

    @os_aws.launch_workers(options[:image_id], options[:instance_type], number_of_instances, worker_options)
  end

  logger.info 'Waiting for server/worker configurations'

  begin
    if @dockerized
      @os_aws.configure_swarm_cluster(@save_directory)
    else
      @os_aws.configure_server_and_workers
    end
  rescue StandardError => e
    raise "Configuring the cluster failed with error `#{e.message}` in:\n#{e.backtrace.join('\n')}"
  end
end
delete_key_pair() click to toggle source

Delete the key pair. Make sure that this happens at the end of whatever you are running, because you will not be able to connect to the instance after you do this.

# File lib/openstudio/aws/aws.rb, line 305
def delete_key_pair
  @os_aws.delete_key_pair
end
describe_all_instances() click to toggle source

List the description of all instances on AWS in the predefined region

# File lib/openstudio/aws/aws.rb, line 351
def describe_all_instances
  @os_aws.describe_all_instances
end
describe_availability_zones() click to toggle source

Return the availability zones for the AWS region specified in the options hash

# File lib/openstudio/aws/aws.rb, line 299
def describe_availability_zones
  @os_aws.describe_availability_zones
end
describe_instances() click to toggle source

Return the description of the instances in the GroupUUID

@example Return Example

[{:instance_id=>"i-45f924ac",
  :image_id=>"ami-845a54ec",
  :state=>{:code=>48, :name=>"terminated"},
  :private_dns_name=>"",
  :public_dns_name=>"",
  :state_transition_reason=>"User initiated (2015-06-01 21:50:40 GMT)",
  :key_name=>"os-key-pair-275a3bf436004c04a1a347ff36337f16",
  :ami_launch_index=>0,
  :product_codes=>[],
  :instance_type=>"m3.medium",
  :launch_time=>2015-06-01 21:13:18 UTC,
  :placement=>
      {:availability_zone=>"us-east-1e", :group_name=>"", :tenancy=>"default"},
      :monitoring=>{:state=>"disabled"},
      :state_reason=>
      {:code=>"Client.UserInitiatedShutdown",
       :message=>"Client.UserInitiatedShutdown: User initiated shutdown"},
      :architecture=>"x86_64",
      :root_device_type=>"ebs",
      :root_device_name=>"/dev/sda1",
      :block_device_mappings=>[],
      :virtualization_type=>"hvm",
      :client_token=>"",
      :tags=>
      [{:key=>"Purpose", :value=>"OpenStudioServer"},
       {:key=>"NumberOfProcessors", :value=>"1"},
       {:key=>"GroupUUID", :value=>"275a3bf436004c04a1a347ff36337f16"},
       {:key=>"Name", :value=>"OpenStudio-Server"},
       {:key=>"UserID", :value=>"unknown_user"}],
      :security_groups=>
      [{:group_name=>"openstudio-server-sg-v2.1", :group_id=>"sg-8740f3ea"}],
      :hypervisor=>"xen",
      :network_interfaces=>[],
      :ebs_optimized=>false}]
# File lib/openstudio/aws/aws.rb, line 346
def describe_instances
  @os_aws.describe_instances
end
determine_image_type(instance_type) click to toggle source
# File lib/openstudio/aws/aws.rb, line 484
def determine_image_type(instance_type)
  image = nil
  if instance_type =~ /cc2/
    image = @default_amis[:cc2worker]
  else
    image = @default_amis[:worker]
  end

  image
end
download_remote_file(server_or_workers, remote_file, local_file) click to toggle source

Download remote files that are on the server or worker. note that the worker at the moment will not work because it would simply overwrite the downloaded filas at this time.

# File lib/openstudio/aws/aws.rb, line 473
def download_remote_file(server_or_workers, remote_file, local_file)
  case server_or_workers
    when :server
      raise 'Server node is nil' unless @os_aws.server

      return @os_aws.server.download_file(remote_file, local_file)
    when :worker
      raise 'Worker file download is not available'
  end
end
estimated_charges() click to toggle source

Return the estimated cost for EC2 instances

# File lib/openstudio/aws/aws.rb, line 361
def estimated_charges
  @os_cloudwatch.estimated_charges
end
group_uuid() click to toggle source

Return the Group UUID as defined in the AWS wrapper

@return [String] UUID

# File lib/openstudio/aws/aws.rb, line 498
def group_uuid
  @os_aws.group_uuid
end
load_instance_info_from_file(filename) click to toggle source
# File lib/openstudio/aws/aws.rb, line 426
def load_instance_info_from_file(filename)
  raise 'Could not find instance description JSON file' unless File.exist? filename

  h = JSON.parse(File.read(filename), symbolize_names: true)
  if h[:location] == 'AWS'
    @os_aws.find_server(h)
  else
    logger.info "Instance file '#{filename}' does not have the location of 'AWS'"
    return false
  end

  true
end
print_connection_info() click to toggle source

Write out to the terminal the connection information for the servers and workers

@return [nil] Only prints to the screen. No return is expected

save_cluster_info(filename) click to toggle source

Save a JSON with information about the cluster that was configured.

@param filename [String] Path and filename to save the JSON file

# File lib/openstudio/aws/aws.rb, line 294
def save_cluster_info(filename)
  File.open(filename, 'w') { |f| f << JSON.pretty_generate(cluster_info) }
end
server() click to toggle source
# File lib/openstudio/aws/aws.rb, line 440
def server
  @os_aws.server
end
shell_command(server_or_workers, command, load_env = true) click to toggle source
# File lib/openstudio/aws/aws.rb, line 458
def shell_command(server_or_workers, command, load_env = true)
  case server_or_workers
    when :server
      raise 'Server node is nil' unless @os_aws.server

      return @os_aws.server.shell_command(command, load_env)
    when :worker
      raise 'Worker list is empty' if @os_aws.workers.empty?

      return @os_aws.workers.each { |w| w.shell_command(command, load_env) }
  end
end
stop() click to toggle source

Stop the entire cluster

# File lib/openstudio/aws/aws.rb, line 366
def stop
  puts "Stoping any instance with group ID: #{@os_aws.group_uuid}"

  stop_instances_by_group_id(@os_aws.group_uuid)
end
stop_instances(group_id, openstudio_instance_type) click to toggle source

Stop running instances

@param group_id [String] The unique group identifier for the OpenStudio cluster. @param openstudio_instance_type [Symbol] The type of instance (:server or :worker)

# File lib/openstudio/aws/aws.rb, line 376
def stop_instances(group_id, openstudio_instance_type)
  instances = @os_aws.describe_running_instances(group_id, openstudio_instance_type.to_sym)
  ids = instances.map { |k, _| k[:instance_id] }

  resp = []
  resp = @os_aws.stop_instances(ids).to_hash unless ids.empty?
  resp
end
stop_instances_by_group_id(group_id) click to toggle source

Warning, it appears that this stops all the instances

# File lib/openstudio/aws/aws.rb, line 386
def stop_instances_by_group_id(group_id)
  instances = @os_aws.describe_running_instances(group_id)
  ids = instances.map { |k, _| k[:instance_id] }

  puts "Stoping the following instances #{ids}"
  resp = []
  resp = @os_aws.stop_instances(ids).to_hash unless ids.empty?
  resp
end
terminate() click to toggle source

Terminate the entire cluster based on the member variable's group_uuid.

# File lib/openstudio/aws/aws.rb, line 420
def terminate
  logger.info "Terminating any instance with GroupUUID: #{@os_aws.group_uuid}"

  terminate_instances_by_group_id(@os_aws.group_uuid)
end
terminate_instances(ids) click to toggle source

@params(ids): array of instance ids

# File lib/openstudio/aws/aws.rb, line 397
def terminate_instances(ids)
  logger.info "Terminating the following instances #{ids}"
  resp = []
  resp = @os_aws.terminate_instances(ids).to_hash unless ids.empty?
  resp
end
terminate_instances_by_group_id(group_id) click to toggle source

Warning, it appears that this terminates all the instances

# File lib/openstudio/aws/aws.rb, line 405
def terminate_instances_by_group_id(group_id)
  raise 'Group ID not defined' unless group_id

  instances = @os_aws.describe_running_instances(group_id)
  logger.info instances
  ids = instances.map { |k, _| k[:instance_id] }

  logger.info "Terminating the following instances #{ids}"
  resp = []
  resp = @os_aws.terminate_instances(ids).to_hash unless ids.empty?

  resp[:terminating_instances].first[:current_state][:name] == 'shutting-down'
end
total_instances_count() click to toggle source

Return the list of all the instances that are running on the account in the availablity zone

# File lib/openstudio/aws/aws.rb, line 356
def total_instances_count
  @os_aws.total_instances_count
end
upload_file(server_or_workers, local_file, remote_file) click to toggle source

Send a file to the server or worker nodes

# File lib/openstudio/aws/aws.rb, line 445
def upload_file(server_or_workers, local_file, remote_file)
  case server_or_workers
    when :server
      raise 'Server node is nil' unless @os_aws.server

      return @os_aws.server.upload_file(local_file, remote_file)
    when :worker
      raise 'Worker list is empty' if @os_aws.workers.empty?

      return @os_aws.workers.each { |w| w.upload_file(local_file, remote_file) }
  end
end

Private Instance Methods

os_aws_file_location() click to toggle source
# File lib/openstudio/aws/aws.rb, line 504
def os_aws_file_location
  # Get the location of the os-aws.rb file.  Use the relative path from where this file exists
  os_aws_file = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'os-aws.rb'))
  raise "os_aws_file does not exist where it is expected: #{os_aws_file}" unless File.exist?(os_aws_file)

  os_aws_file
end