module Seiya
Constants
- VERSION
Public Instance Methods
component_instance_variables(*variable_names)
click to toggle source
# File lib/seiya.rb, line 83 def component_instance_variables(*variable_names) variable_names.each do |variable_name| variable_name = variable_name.to_s const_name = variable_name.upcase super_class = Seiya.const_get variable_name.sub(/s$/, '').camelize vars = Settings.const_get const_name begin vars.merge! Util.get_const "#{@settings_const_str}::#{const_name}" rescue NameError # ignored end vars = {} unless vars.is_a? Hash vars = vars.select do |_, v| v >= 0 end.sort_by do |_, v| v end.to_h vars = vars.keys.map do |k| require_str, const_str = k.split '|' klass = get_const require_str, const_str klass.new end.select do |p| p.is_a? super_class end instance_variable_set '@' << variable_name, vars end end
extend_load_path(path)
click to toggle source
# File lib/seiya.rb, line 78 def extend_load_path(path) load_path = get_load_path path $:.unshift load_path unless load_path.nil? end
gen_project_file(project_name)
click to toggle source
# File lib/seiya.rb, line 243 def gen_project_file(project_name) base_path = "#{project_name}/#{project_name}" FileUtils.mkpath base_path FileUtils.mkpath "#{base_path}/tasks" FileUtils.mkpath "#{base_path}/items" FileUtils.mkpath "#{base_path}/pipelines" File.write("#{project_name}/seiya.ini", %([global] settings = #{project_name}/settings|Settings )) File.write("#{base_path}/settings.rb", %(module Settings PIPELINES = { '#{project_name}/pipelines|Pipelines::Test' => 10, } end )) File.write("#{base_path}/items.rb", "require 'items/test'\n") File.write("#{base_path}/pipelines.rb", "require 'pipelines/test'\n") File.write("#{base_path}/tasks.rb", "require 'tasks/test'\n") File.write("#{base_path}/items/test.rb", %(require 'seiya' module Items class Test < Seiya::Item def to_s inspect end end end )) File.write("#{base_path}/pipelines/test.rb", %(require 'seiya' module Pipelines class Test < Seiya::Pipeline def process_item(item) puts 'I am in Test pipeline!' item.to_s puts item item end end end )) File.write("#{base_path}/tasks/test.rb", %(require 'seiya' require_relative '../items' module Tasks class Test < Seiya::Task def initialize @start_urls = ['http://www.baidu.com/'] end def parse(response, enum) item = Items::Test.new url: response.url enum.yield item end end end )) puts "New Seiya project '#{project_name}' created in: #{File.join(Dir.pwd, project_name)} You can start your first task with: cd #{project_name} seiya gentask example example.com" end
gen_task_file(task_name, task_domain = nil)
click to toggle source
# File lib/seiya.rb, line 208 def gen_task_file(task_name, task_domain = nil) load_path = get_load_path Dir.pwd if load_path.nil? puts 'Please in a seiya project directory!' exit! end File.open File.join(load_path, 'tasks.rb'), 'a' do |file| file.puts "require 'tasks/#{task_name}'" end task_dir = File.join load_path, 'tasks' unless File.exist? task_dir FileUtils.mkpath task_dir end task_file_name = "#{File.join(task_dir, task_name)}.rb" if File.exist? task_file_name puts "task file: #{task_file_name} exist!" exit! end File.write(task_file_name, %(require 'seiya' module Tasks class #{task_name.camelize} < Seiya::Task def initialize @start_urls = [#{task_domain.nil? ? '' : "'#{task_domain}'"}] end def parse(response, enum) end end end )) end
get_classes(const_path, super_class = Object)
click to toggle source
# File lib/seiya.rb, line 44 def get_classes(const_path, super_class = Object) require_str, const_str = const_path.split '|' begin _module = get_const require_str, const_str rescue LoadError, NameError return [] end _module.constants.select do |c| const = _module.const_get(c) const.is_a? Class and const < super_class end.map do |c| _module.const_get c end end
get_const(require_str, const_str)
click to toggle source
# File lib/seiya.rb, line 36 def get_const(require_str, const_str) if const_str.nil? return Util.get_const require_str end require require_str Util.get_const const_str end
get_const!(require_str, const_str)
click to toggle source
# File lib/seiya.rb, line 19 def get_const!(require_str, const_str) begin require require_str rescue LoadError => e puts e puts "Cannot load #{require_str}" exit! end begin Util.get_const const_str rescue NameError puts "Cannot get #{const_str}" exit! end end
get_load_path(path)
click to toggle source
# File lib/seiya.rb, line 60 def get_load_path(path) Dir.foreach path do |f| if %w(. ..).include? f next end new_path = File.join path, f unless File.directory? new_path next end Dir.foreach new_path do |_f| if _f == 'tasks.rb' return new_path end end end nil end
get_task_class(task_name)
click to toggle source
# File lib/seiya.rb, line 194 def get_task_class(task_name) if @task_classes.nil? setup end task_name = task_name.to_sym unless @task_classes.key? task_name puts "Task #{task_name} does not exist!" exit! end @task_classes[task_name] end
process_item(item)
click to toggle source
# File lib/seiya/processer.rb, line 4 def process_item(item) @pipelines.each do |p| item = p.process_item item end end
process_request(request)
click to toggle source
# File lib/seiya/processer.rb, line 10 def process_request(request) @request_middlewares.each do |rm| rm.process_request request end end
run_command(command, *args)
click to toggle source
# File lib/seiya.rb, line 160 def run_command(command, *args) if @commands.nil? setup end command_sym = command.to_sym unless @commands.key? command_sym puts "No command: #{command}" exit! end @commands[command_sym].run! *args end
setup(conf_file: 'seiya.ini')
click to toggle source
# File lib/seiya.rb, line 115 def setup(conf_file: 'seiya.ini') settings_const_str = '' if File.exist? conf_file path = File.dirname File.expand_path(conf_file) extend_load_path path conf = IniFile.load conf_file settings_file = conf.to_h.fetch('global', {}).fetch('settings', 'settings') settings_require_str, settings_const_str = settings_file.split '|' @settings_const_str = settings_const_str require settings_require_str end component_instance_variables :pipelines, :request_middlewares commands = [Settings::COMMANDS] begin commands << Util.get_const("#{settings_const_str}::COMMANDS") rescue NameError # ignored end @commands = commands.map do |const_path| command_classes = get_classes(const_path, Command) command_classes.map do |klass| [klass.name.underscore.split('/').last.to_sym, klass.new] end end.flatten(1).to_h task_classes = get_classes('tasks|Tasks', Task) @task_classes = task_classes.map do |klass| [klass.name.underscore.split('/').last.to_sym, klass] end.to_h end
tasks()
click to toggle source
# File lib/seiya.rb, line 150 def tasks if @task_classes.nil? setup end @task_classes.map do |k, v| [k, v.summary] end.to_h end
usage()
click to toggle source
# File lib/seiya.rb, line 174 def usage if @commands.nil? setup end available_commands = @commands.map do |k, v| ' %-14s%-30s' % [k.to_s, v.summary] end.join("\n") puts %(Seiya #{VERSION} Usage: seiya <command> [options] [args] Available commands: #{available_commands} Use "seiya <command> -h" to see more info about a command ) end