class Dumper::Profiles::SankakuComplex
Public Instance Methods
dump(url, path, from, to)
click to toggle source
# File lib/dumper/profiles/sankakucomplex.rb, line 24 def dump(url, path, from, to) if url.include?('idol.sankakucomplex') || url.include?('chan.sankakucomplex') to = 1 if to == -1 prefix = url.include?('idol.sankakucomplex') ? 'idol' : 'chan' from.upto(to) { |page| u = "#{url}&page=#{page}" begin op = open u rescue Exception => e sleep 1 retry end Nokogiri::HTML(op).xpath('//a/@href').each { |p| next unless p.to_s.start_with? '/post/show' errors = 0 @pool.process { begin img = Nokogiri::HTML(open("http://#{prefix}.sankakucomplex.com/#{p}")).at_xpath('//a[@itemprop="contentUrl"]/@href').to_s Dumper.get path, img, { referer: u } rescue Exception => e if errors <= 3 sleep 3 errors += 1 retry end end } } } else from -= 1 to -= 1 if to >= 1 [].tap { |urls| Nokogiri::HTML(open(url)).xpath('//a[@target="_blank"]/@href').each { |u| urls << u if u.to_s.start_with? 'http://images.sankakucomplex.com/wp-content/' } }[from..to].each { |p| @pool.process { Dumper.get path, p, { referer: url } } } end end