class Dumper::Profiles::SankakuComplex

Public Instance Methods

dump(url, path, from, to) click to toggle source
# File lib/dumper/profiles/sankakucomplex.rb, line 24
def dump(url, path, from, to)
  if url.include?('idol.sankakucomplex') || url.include?('chan.sankakucomplex')
    to     = 1 if to == -1
    prefix = url.include?('idol.sankakucomplex') ? 'idol' : 'chan'

    from.upto(to) { |page|
      u = "#{url}&page=#{page}"
      begin
        op = open u
      rescue Exception => e
        sleep 1
        retry
      end

      Nokogiri::HTML(op).xpath('//a/@href').each { |p|
        next unless p.to_s.start_with? '/post/show'
        errors = 0

        @pool.process {
          begin
            img = Nokogiri::HTML(open("http://#{prefix}.sankakucomplex.com/#{p}")).at_xpath('//a[@itemprop="contentUrl"]/@href').to_s
            Dumper.get path, img, { referer: u }
          rescue Exception => e
            if errors <= 3
              sleep 3
              errors += 1
              retry
            end
          end
        }
      }
    }
  else
    from -= 1
    to   -= 1 if to >= 1

    [].tap { |urls|
      Nokogiri::HTML(open(url)).xpath('//a[@target="_blank"]/@href').each { |u|
        urls << u if u.to_s.start_with? 'http://images.sankakucomplex.com/wp-content/'
      }
    }[from..to].each { |p|
      @pool.process {
        Dumper.get path, p, { referer: url }
      }
    }
  end
end