module Mobilize::Ssh
Constants
- VERSION
Public Class Methods
config()
click to toggle source
# File lib/mobilize-ssh/helpers/ssh_helper.rb, line 3 def self.config Base.config('ssh') end
default_node()
click to toggle source
# File lib/mobilize-ssh/helpers/ssh_helper.rb, line 23 def self.default_node self.nodes.first end
deploy(node,user_name,unique_name,command,file_hash)
click to toggle source
# File lib/mobilize-ssh/handlers/ssh.rb, line 17 def Ssh.deploy(node,user_name,unique_name,command,file_hash) loc_dir = Ssh.pop_loc_dir(unique_name,file_hash) Ssh.fire!(node,"rm -rf #{unique_name} && mkdir -p #{unique_name} && chown -R #{Ssh.node_owner(node)} #{unique_name}") if loc_dir Ssh.scp(node,loc_dir,".") #make sure loc_dir is removed FileUtils.rm_r(loc_dir,:force=>true) end #create cmd_file in unique_name cmd_path = "#{unique_name}/cmd.sh" Ssh.write(node,command,cmd_path) #move folder to user's home, change ownership user_dir = "/home/#{user_name}/" mobilize_dir = "#{user_dir}mobilize/" deploy_dir = "#{mobilize_dir}#{unique_name}/" deploy_cmd_path = "#{deploy_dir}cmd.sh" deploy_cmd = "sudo mkdir -p #{mobilize_dir} && " + "sudo rm -rf #{mobilize_dir}#{unique_name} && " + "sudo mv #{unique_name} #{mobilize_dir} && " + "sudo chown -R #{user_name} #{mobilize_dir} && " + "sudo chmod -R 0700 #{user_name} #{mobilize_dir}" Ssh.fire!(node,deploy_cmd) #need to use bash or we get no tee full_cmd = "/bin/bash -l -c '(cd #{deploy_dir} && sh #{deploy_cmd_path} > >(tee stdout) 2> >(tee stderr >&2))'" #fire_cmd runs sh on cmd_path, optionally with sudo su fire_cmd = %{sudo su #{user_name} -c "#{full_cmd}"} return fire_cmd end
file_hash_by_stage_path(stage_path,gdrive_slot)
click to toggle source
# File lib/mobilize-ssh/handlers/ssh.rb, line 188 def Ssh.file_hash_by_stage_path(stage_path,gdrive_slot) file_hash = {} s = Stage.where(:path=>stage_path).first u = s.job.runner.user user_name = Ssh.user_name_by_stage_path(stage_path) s.sources(gdrive_slot).each do |sdst| split_path = sdst.path.split("/") #if path is to stage output, name with stage name file_name = if (split_path.last == "out" and (1..5).to_a.map{|n| "stage#{n.to_s}"}.include?(split_path[-2].to_s)) #<jobname>/stage1/out "#{split_path[-2]}.out" elsif (1..5).to_a.map{|n| "stage#{n.to_s}"}.include?(split_path.last[-6..-1]) #runner<jobname>stage1 "#{split_path.last[-6..-1]}.out" else split_path.last end if ["gsheet","gfile"].include?(sdst.handler) #google drive sources are always read as the user #with the apportioned slot file_hash[file_name] = sdst.read(u.name,gdrive_slot) else #other sources should be read by su-user file_hash[file_name] = sdst.read(user_name) end end return file_hash end
fire!(node,cmd)
click to toggle source
# File lib/mobilize-ssh/handlers/ssh.rb, line 127 def Ssh.fire!(node,cmd) puts "#{Time.now.utc}--Ssh on #{node}: #{cmd}" name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}} key_path = "#{Base.root}/#{key}" opts = {:port=>(port || 22),:keys=>key_path} response = if Ssh.needs_gateway?(node) gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}} gkey_path = "#{Base.root}/#{gkey}" gopts = {:port=>(gport || 22),:keys=>gkey_path} Net::SSH::Gateway.run(gname,guser,name,user,cmd,gopts,opts) else Net::SSH.start(name,user,opts) do |ssh| ssh.run(cmd) end end response end
gateway(node)
click to toggle source
# File lib/mobilize-ssh/helpers/ssh_helper.rb, line 11 def self.gateway(node) self.config['nodes'][node]['gateway'] end
home_dir()
click to toggle source
# File lib/mobilize-ssh.rb, line 12 def Ssh.home_dir File.expand_path('..',File.dirname(__FILE__)) end
host(node)
click to toggle source
# File lib/mobilize-ssh/helpers/ssh_helper.rb, line 7 def self.host(node) self.config['nodes'][node]['host'] end
needs_gateway?(node)
click to toggle source
determine if current machine is on host domain, needs gateway if one is provided and it is not
# File lib/mobilize-ssh/helpers/ssh_helper.rb, line 36 def self.needs_gateway?(node) return false if self.skip_gateway?(node) begin host_domain_name = self.host(node)['name'].split(".")[-2..-1].join(".") return true if self.gateway(node) and Socket.domain_name != host_domain_name rescue return false end end
node_owner(node)
click to toggle source
# File lib/mobilize-ssh/helpers/ssh_helper.rb, line 27 def self.node_owner(node) self.host(node)['user'] end
nodes()
click to toggle source
# File lib/mobilize-ssh/helpers/ssh_helper.rb, line 19 def self.nodes self.config['nodes'].keys end
path_to_dst(path,stage_path,gdrive_slot)
click to toggle source
converts a source path or target path to a dst in the context of handler and stage
# File lib/mobilize-ssh/handlers/ssh.rb, line 47 def Ssh.path_to_dst(path,stage_path,gdrive_slot) has_handler = true if path.index("://") red_path = path.split("://").last #is user has a handler, their first path node is a node name, #or there are more than 2 path nodes, try to find Ssh file if has_handler or Ssh.nodes.include?(red_path.split("/").first) or red_path.split("/").length > 2 user_name = Ssh.user_name_by_stage_path(stage_path) ssh_url = Ssh.url_by_path(red_path,user_name) return Dataset.find_or_create_by_url(ssh_url) end #otherwise, use Gsheet return Gsheet.path_to_dst(red_path,stage_path,gdrive_slot) end
pop_loc_dir(unique_name,file_hash)
click to toggle source
# File lib/mobilize-ssh/handlers/ssh.rb, line 5 def Ssh.pop_loc_dir(unique_name,file_hash) loc_dir = "/tmp/#{unique_name}" `rm -rf #{loc_dir} && mkdir -p #{loc_dir}` file_hash.each do |fname,fdata| fpath = "#{loc_dir}/#{fname}" #for now, only gz is binary mode = fname.ends_with?(".gz") ? "wb" : "w" File.open(fpath,mode) {|f| f.print(fdata)} end return loc_dir if file_hash.keys.length>0 end
read_by_dataset_path(dst_path,user_name,*args)
click to toggle source
# File lib/mobilize-ssh/handlers/ssh.rb, line 145 def Ssh.read_by_dataset_path(dst_path,user_name,*args) #expects node as first part of path node,path = dst_path.split("/").ie{|pa| [pa.first,pa[1..-1].join("/")]} #slash in front of path response = Ssh.run(node,"cat /#{path}",user_name) if response['exit_code'] == 0 return response['stdout'] else raise "Unable to read ssh://#{dst_path} with error: #{response['stderr']}" end end
run(node,command,user_name,stage_path=nil,file_hash={},run_params=nil)
click to toggle source
# File lib/mobilize-ssh/handlers/ssh.rb, line 100 def Ssh.run(node,command,user_name,stage_path=nil,file_hash={},run_params=nil) file_hash ||= {} run_params ||={} #replace any params in the file_hash and command run_params.each do |k,v| command.gsub!("@#{k}",v) file_hash.each do |name,data| data.gsub!("@#{k}",v) end end #make sure the dir for this command is unique unique_name = if stage_path stage_path.downcase.alphanunderscore else [user_name,node,command,file_hash.keys.to_s,Time.now.to_f.to_s].join.to_md5 end fire_cmd = Ssh.deploy(node, user_name, unique_name, command, file_hash) result = Ssh.fire!(node,fire_cmd) #clear out the md5 folders and those not requested to keep s = Stage.find_by_path(stage_path) if stage_path unless s and s.params['save_logs'] rm_cmd = "sudo rm -rf /home/#{user_name}/mobilize/#{unique_name}" Ssh.fire!(node,rm_cmd) end return result end
run_by_stage_path(stage_path)
click to toggle source
# File lib/mobilize-ssh/handlers/ssh.rb, line 217 def Ssh.run_by_stage_path(stage_path) gdrive_slot = Gdrive.slot_worker_by_path(stage_path) #return blank response if there are no slots available return nil unless gdrive_slot s = Stage.where(:path=>stage_path).first u = s.job.runner.user params = s.params node, command = [params['node'],params['cmd']] node ||= Ssh.default_node user_name = Ssh.user_name_by_stage_path(stage_path) #do not allow server commands from non-sudoers for the special server node if node=='server' and !Ssh.sudoers(node).include?(u.name) raise "You do not have permission to run commands on the mobilize server" end file_hash = Ssh.file_hash_by_stage_path(stage_path,gdrive_slot) Gdrive.unslot_worker_by_path(stage_path) run_params = params['params'] result = Ssh.run(node,command,user_name,stage_path,file_hash,run_params) #use Gridfs to cache result response = {} response['out_url'] = Dataset.write_by_url("gridfs://#{s.path}/out",result['stdout'].to_s,Gdrive.owner_name) response['err_url'] = Dataset.write_by_url("gridfs://#{s.path}/err",result['stderr'].to_s,Gdrive.owner_name) if result['stderr'].to_s.length>0 #is an error if there is no out and there is an err, regardless of signal result['exit_code'] = 500 if result['stdout'].to_s.strip.length==0 and result['stderr'].to_s.strip.length>0 response['signal'] = result['exit_code'] response end
scp(node,from_path,to_path)
click to toggle source
# File lib/mobilize-ssh/handlers/ssh.rb, line 83 def Ssh.scp(node,from_path,to_path) name,key,port,user = Ssh.host(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}} key_path = "#{Base.root}/#{key}" opts = {:port=>(port || 22),:keys=>key_path} if Ssh.needs_gateway?(node) gname,gkey,gport,guser = Ssh.gateway(node).ie{|h| ['name','key','port','user'].map{|k| h[k]}} gkey_path = "#{Base.root}/#{gkey}" gopts = {:port=>(gport || 22),:keys=>gkey_path} return Net::SSH::Gateway.sync(gname,guser,name,user,from_path,to_path,gopts,opts) else Net::SCP.start(name,user,opts) do |scp| scp.upload!(from_path,to_path,:recursive=>true) end end return true end
skip_gateway?(node)
click to toggle source
# File lib/mobilize-ssh/helpers/ssh_helper.rb, line 31 def self.skip_gateway?(node) self.config['nodes'][node]['skip_gateway'] end
sudoers(node)
click to toggle source
# File lib/mobilize-ssh/helpers/ssh_helper.rb, line 15 def self.sudoers(node) self.config['nodes'][node]['sudoers'] end
tmp_file(fdata,binary=false,fpath=nil)
click to toggle source
# File lib/mobilize-ssh/handlers/ssh.rb, line 165 def Ssh.tmp_file(fdata,binary=false,fpath=nil) #creates a file under tmp/files with an md5 from the data tmp_file_path = fpath || "#{Dir.mktmpdir}/#{(fdata + Time.now.utc.to_f.to_s).to_md5}" write_mode = binary ? "wb" : "w" #write data to path File.open(tmp_file_path,write_mode) {|f| f.print(fdata)} return tmp_file_path end
url_by_path(path,user_name)
click to toggle source
# File lib/mobilize-ssh/handlers/ssh.rb, line 61 def Ssh.url_by_path(path,user_name) node = path.split("/").first.to_s if Ssh.nodes.include?(node) #cut node out of path path = "/" + path.split("/")[1..-1].join("/") else node = Ssh.default_node path = path.starts_with?("/") ? path : "/#{path}" end url = "ssh://#{node}#{path}" begin response = Ssh.run(node, "head -1 #{path}", user_name) if response['exit_code'] != 0 raise "Unable to find #{url} with error: #{response['stderr']}" else return "ssh://#{node}#{path}" end rescue => exc raise Exception, "Unable to find #{url} with error: #{exc.to_s}", exc.backtrace end end
user_name_by_stage_path(stage_path,node=nil)
click to toggle source
# File lib/mobilize-ssh/handlers/ssh.rb, line 174 def Ssh.user_name_by_stage_path(stage_path,node=nil) s = Stage.where(:path=>stage_path).first u = s.job.runner.user user_name = s.params['user'] node = s.params['node'] node = Ssh.default_node unless Ssh.nodes.include?(node) if user_name and !Ssh.sudoers(node).include?(u.name) raise "#{u.name} does not have su permissions for this node" elsif user_name.nil? user_name = u.name end return user_name end
write(node,fdata,to_path,binary=false)
click to toggle source
# File lib/mobilize-ssh/handlers/ssh.rb, line 157 def Ssh.write(node,fdata,to_path,binary=false) from_path = Ssh.tmp_file(fdata,binary) Ssh.scp(node,from_path,to_path) #make sure local is removed FileUtils.rm_r(from_path,:force=>true) return true end