class Object
Public Class Methods
execute_cmd(cmd, timeout, log_out, log_err)
click to toggle source
# File lib/socotra-build.rb, line 80 def self.execute_cmd(cmd, timeout, log_out, log_err) pid = spawn(cmd, :out=>log_out, :err=>log_err) begin return Timeout::timeout(timeout) do _, status = Process.wait2(pid) return status.success? end rescue Timeout::Error while pid_exists(pid) if kill_pid(pid, timeout) return false end end end end
generate_aws_keys(key, log_identifier)
click to toggle source
# File lib/socotra-build.rb, line 182 def self.generate_aws_keys(key, log_identifier) cmd = "vault read #{key}" status, output = system_safe(cmd, log_identifier, "AWS key generation failed", true, false) cmd = "echo '#{output}' |grep access_key|awk '{print $2'}" status, account_access_key = system_safe(cmd, "get_generated_access_key", "Getting account access key failed", true, false, false) account_access_key = account_access_key.strip cmd = "echo '#{output}' |grep secret_key|awk '{print $2'}" status, account_secret_key = system_safe(cmd, "get_generated_access_key", "Getting account access key failed", true, false, false) account_secret_key = account_secret_key.strip cmd = "echo '#{output}' |grep lease_id|awk '{print $2'}" status, lease_id = system_safe(cmd, "get_generated_access_key", "Getting account access key failed", true, false, false) lease_id = lease_id.strip return account_access_key, account_secret_key, lease_id end
get_sanitized_cmd(cmd)
click to toggle source
# File lib/socotra-build.rb, line 27 def self.get_sanitized_cmd(cmd) secret_fields = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "password", "aws_access_key_id", "aws_secret_access_key", "aws_access_key", "aws_secret_key", "access_key", "secret_key", "account_access_key", "account_secret_key", "aws.accessKeyId", "aws.secretKey", "jwtsecret", "master_access_key", "master_secret_key", "github_password"] cmd = cmd.gsub(/\s+/m, ' ').strip.split(" ") replace = cmd.grep(Regexp.union(secret_fields)) unless replace.empty? replace = Set.new replace cmd.collect! {|e| (replace.include? e) ? '<secret>': e} end cmd = cmd.join(' ') end
get_secret(key, subkey, log_identifier)
click to toggle source
# File lib/socotra-build.rb, line 170 def self.get_secret(key, subkey, log_identifier) secret = `vault read -field=#{subkey} #{key}` secret = secret.strip return secret end
get_url(url)
click to toggle source
# File lib/tenant.rb, line 9 def self.get_url(url) timeout = 1200 c = HTTPClient.new c.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE c.ssl_config.timeout = timeout connection_exceptions = [HTTPClient::BadResponseError, HTTPClient::ConnectTimeoutError] begin ssl_tries ||= timeout dns_tries ||= timeout http_tries ||= timeout content = c.get_content(url) rescue OpenSSL::SSL::SSLError if (ssl_tries -= 1) > 0 sleep 1 retry else puts "ERROR: ELB is up, but instances are not (SSL)" $build_failure = true end rescue *connection_exceptions if (http_tries -= 1) > 0 sleep 1 retry else puts "ERROR: ELB is up, but instances are not (HTTP)" $build_failure = true end rescue SocketError if (dns_tries -= 1) > 0 sleep 1 retry else puts "ERROR: DNS record not created" $build_failure = true end else return content end end
install_socotra_py_modules(pymodule, environment)
click to toggle source
# File lib/socotra-build.rb, line 208 def self.install_socotra_py_modules(pymodule, environment) cmd = "sudo pip list |grep #{pymodule}; if [ $? -eq 0 ];then /usr/bin/yes|sudo pip uninstall #{pymodule};fi" system_safe(cmd, "uninstall_#{pymodule}_py_module", "Failed to uninstall #{pymodule}") cmd = "/usr/bin/yes|sudo pip install --egg --process-dependency-links --allow-external mysql-connector-python --trusted-host socotra-pypi-euw1.s3-website-eu-west-1.amazonaws.com --extra-index-url=http://socotra-pypi-euw1.s3-website-eu-west-1.amazonaws.com/#{pymodule}/#{environment} #{pymodule}" system_safe(cmd, "install_#{pymodule}_py_module", "Failed to install #{pymodule}") end
kill_pid(pid, timeout)
click to toggle source
# File lib/socotra-build.rb, line 66 def self.kill_pid(pid, timeout) puts "INFO: #{pid} timed out after #{timeout}, killing it" killpid = fork { Process.kill('INT', pid) } Process.detach(killpid) begin Process.waitpid(killpid) rescue Errno::ECHILD return true end return false end
load_assets_from_branch(tenant_name, domain, admin_username, admin_password, api_url, apidoc_url)
click to toggle source
# File lib/tenant.rb, line 86 def self.load_assets_from_branch(tenant_name, domain, admin_username, admin_password, api_url, apidoc_url) if api_url == "https://api.staging.socotra.com" cmd = "socotraadmin tenant add_tenant #{tenant_name} --overwrite_ontology --tenant_hostname=#{tenant_name}.co.#{domain} --tenant_path=. --admin_username=#{admin_username} --admin_password=#{admin_password} --apidoc_url=#{apidoc_url} --api_url=#{api_url}" else cmd = "socotraadmin tenant add_tenant #{tenant_name} --tenant_hostname=#{tenant_name}.co.#{domain} --tenant_path=. --admin_username=#{admin_username} --admin_password=#{admin_password} --apidoc_url=#{apidoc_url} --api_url=#{api_url}" end system_safe(cmd, "load_branch_assets_for_pr", "Failed to load prod asset for PR") end
load_production_assets(version, github_username, github_password, tenant_name, domain, socotra_username, socotra_password, api_url, apidoc_url)
click to toggle source
# File lib/tenant.rb, line 50 def self.load_production_assets(version, github_username, github_password, tenant_name, domain, socotra_username, socotra_password, api_url, apidoc_url) url = "https://github.com/socotra/prime/archive/#{version}.tar.gz" assets_dir = Dir.mktmpdir assets_artifact = "#{assets_dir}/#{version}.tar.gz" open(assets_artifact, "w").write(open(url, :http_basic_authentication => [github_username, github_password]).read) tar_extract = Gem::Package::TarReader.new(Zlib::GzipReader.open(assets_artifact)) tar_extract.rewind tar_extract.each do |entry| if entry.file? FileUtils.mkdir_p(File.dirname("#{assets_dir}/#{entry.full_name}")) File.open("#{assets_dir}/#{entry.full_name}", "wb") do |f| f.write(entry.read) end File.chmod(entry.header.mode, "#{assets_dir}/#{entry.full_name}") end end tar_extract.close cmd = "socotraadmin tenant add_tenant #{tenant_name} --tenant_hostname=#{tenant_name}.co.#{domain} --tenant_path=#{assets_dir}/prime-#{version} --overwrite_ontology --admin_username=#{socotra_username} --admin_password=#{socotra_password} --apidoc_url=#{apidoc_url} --api_url=#{api_url}" system_safe(cmd, "load_prod_assets_for_pr", "Failed to load prod asset for PR") end
load_tabular_datasources(jwtsecret, api_url, apidoc_url, admin_username, admin_password, tenant_name)
click to toggle source
# File lib/tenant.rb, line 74 def self.load_tabular_datasources(jwtsecret, api_url, apidoc_url, admin_username, admin_password, tenant_name) cmd = "socotraadmin tabular_datasource create_bulk \ --jwtsecret=#{$jwtsecret} \ --api_url=#{$api_url} \ --apidoc_url=#{$apidoc_url} \ --admin_username=#{$admin_username} \ --admin_password=#{$admin_password} \ #{$tenant_name} \ $(pwd)/data/tables.json" system_safe(cmd, "load_tabular_datasources", "Failed to load tabular datasources") end
load_tasks(version, environment, tenant_name)
click to toggle source
# File lib/tenant.rb, line 96 def self.load_tasks(version, environment, tenant_name) cmd = "sg docker -c \'tasker --path=reports --tag=#{$version} --environment=#{$environment} --timezone='Africa/Kigali' --tenant=#{$tenant_name} -v\'" system_safe(cmd, "load_tasks", "Failed to load task for PR") end
log(level, message)
click to toggle source
# File lib/socotra-build.rb, line 96 def self.log(level, message) date = Time.now.strftime("%Y-%m-%d %H:%M:%S") puts "#{date} #{level}: #{message}" end
logger(log, success, retry_exec, raise_on_fail, log_out, log_err, error_message, last_try)
click to toggle source
# File lib/socotra-build.rb, line 101 def self.logger(log, success, retry_exec, raise_on_fail, log_out, log_err, error_message, last_try) if log log("INFO", "Logs available here: #{log_err} and #{log_out}" ) else log("INFO", "Logs unavailable due to user settings") end if not (success or retry_exec) log("ERROR", error_message) raise error_message end if not success and retry_exec if last_try and raise_on_fail log_level = "ERROR" else log_level = "WARNING" end if log log(log_level, "see #{log_out} and #{log_err} for details") end log(log_level, error_message) if not last_try raise error_message end end end
pid_exists(pid)
click to toggle source
# File lib/socotra-build.rb, line 56 def self.pid_exists(pid) begin Process.getpgid(pid) rescue Errno::ESRCH return false end return true end
put_secret(key, subkey_value_map, log_identifier)
click to toggle source
# File lib/socotra-build.rb, line 177 def self.put_secret(key, subkey_value_map, log_identifier) cmd = "vault write #{key} #{subkey_value_map}" system_safe(cmd, log_identifier, "Put secret failed", true, false) end
revoke_lease(lease_id, log_identifier)
click to toggle source
# File lib/socotra-build.rb, line 203 def self.revoke_lease(lease_id, log_identifier) cmd = "vault revoke #{lease_id}" safe_retry(cmd, log_identifier, "Revoking lease failed", false, "ERROR", false) end
safe_retry(cmd, description, error_message="command failed", raise_on_fail=true, log=true, tries=3, timeout=0)
click to toggle source
# File lib/socotra-build.rb, line 6 def self.safe_retry(cmd, description, error_message="command failed", raise_on_fail=true, log=true, tries=3, timeout=0) retry_count = 0 last_try = false begin if (retry_count == tries - 1) last_try = true end status, output = system_safe(cmd, description, error_message, raise_on_fail=raise_on_fail, log=log, print_command=true, retry_exec=true, timeout=timeout, log_file_number=retry_count, last_try=last_try) rescue sleep 5 retry if (retry_count += 1) < (tries) end if last_try and raise_on_fail and status != 0 raise error_message else return status end end
sync_logs(repo, build_id)
click to toggle source
# File lib/socotra-build.rb, line 216 def self.sync_logs(repo, build_id) cmd = "cp -f console.log logs/" system_safe(cmd, "copy_console_log", "Failed to copy console log") upload_to_s3("logs", "socotra-builds/#{repo}/#{build_id}") end
system_safe(cmd=cmd, cmd_description=cmd_description, error_message=error_message, raise_on_fail=true, log=true, print_command=true, retry_exec=false, timeout=0, log_file_number=1, last_try=true)
click to toggle source
# File lib/socotra-build.rb, line 133 def self.system_safe(cmd=cmd, cmd_description=cmd_description, error_message=error_message, raise_on_fail=true, log=true, print_command=true, retry_exec=false, timeout=0, log_file_number=1, last_try=true) start = Time.now.to_i cmd_sanitized = get_sanitized_cmd(cmd) if print_command log("INFO", "Executing command: #{cmd_sanitized}") else log("INFO", "Not logging command") end if log log_out = "logs/#{cmd_description}_out.log.#{log_file_number}" log_err = "logs/#{cmd_description}_err.log.#{log_file_number}" else log_out = "/dev/null" log_err = "/dev/null" end success = execute_cmd(cmd, timeout, log_out, log_err) elapsed = Time.now.to_i - start log("INFO", "#{cmd_description} took #{elapsed} seconds") file = File.open(log_out, "rb") out = file.read logger(log, success, retry_exec, raise_on_fail, log_out, log_err, error_message, last_try) return success end
system_unsafe(cmd)
click to toggle source
# File lib/socotra-build.rb, line 165 def self.system_unsafe(cmd) success = system(cmd) return success end
Public Instance Methods
copy_directory(s3_source, s3_destination)
click to toggle source
# File lib/s3_uploader.rb, line 19 def copy_directory(s3_source, s3_destination) cmd = "#{get_path_inject} aws s3 sync s3://#{s3_source} s3://#{s3_destination} --delete --acl public-read" source_sanitized = sanitize_for_filename(s3_source) destination_sanitized = sanitize_for_filename(s3_destination) system_safe(cmd, "s3_copy_#{source_sanitized}_to_#{destination_sanitized}", "distribution failed on aws copy") puts "distribution success to s3: #{s3_destination}" end
get_path_inject()
click to toggle source
# File lib/s3_uploader.rb, line 1 def get_path_inject return path_inject = "AWS_DEFAULT_REGION=\"eu-west-1\" " + "AWS_ACCESS_KEY_ID=\"#{$master_access_key}\" " + "AWS_SECRET_ACCESS_KEY=\"#{$master_secret_key}\" " end
sanitize_for_filename(path)
click to toggle source
# File lib/s3_uploader.rb, line 7 def sanitize_for_filename(path) return path.tr("/", "_") end
upload_to_s3(source_dir, s3_destination)
click to toggle source
# File lib/s3_uploader.rb, line 11 def upload_to_s3 (source_dir, s3_destination) cmd_sync = "#{get_path_inject} aws s3 sync #{source_dir} s3://#{s3_destination} --acl public-read" source_sanitized = sanitize_for_filename(source_dir) destination_sanitized = sanitize_for_filename(s3_destination) system_safe(cmd_sync, "s3_upload_#{source_sanitized}_to_#{destination_sanitized}", "distribution failed on aws sync") puts "distribution success to s3: #{s3_destination}" end