class TD2Planet::Fetcher

Public Class Methods

new(cache_dir, dry_run=false, user_agent="td2planet") click to toggle source
   # File lib/td2planet/fetcher.rb
12 def initialize(cache_dir, dry_run=false, user_agent="td2planet")
13   @cache_dir = Pathname.new(cache_dir)
14   unless @cache_dir.exist?
15     @cache_dir.mkdir
16   end
17   @dry_run = dry_run
18   @user_agent = user_agent
19 end

Public Instance Methods

fetch_all_rss(uris) click to toggle source
   # File lib/td2planet/fetcher.rb
21 def fetch_all_rss(uris)
22   rss_list = []
23   uris.each do |uri|
24     cache_file = @cache_dir + ERB::Util.u(uri)
25     if @dry_run
26       puts "use cache: #{cache_file}"
27       text = cache_file.read
28     else
29       text = nil
30       begin
31         puts "fetch: #{uri}"
32         text = URI.parse(uri).read("User-Agent" => @user_agent)
33       rescue Timeout::Error
34         # fallback
35         puts "ERROR: timeout #{uri}"
36         text = cache_file.read
37       rescue Exception
38         puts "ERROR: #{$!} (#{$!.class}) on #{uri}"
39         next
40       else
41         if text.status[0] == '200' && /rss/ =~ text
42           cache_file.open('wb'){|f| f.write(text) }
43         else
44           # fallback
45           puts "ERROR: fetch failed #{uri} #{text.status}"
46           text = cache_file.read
47         end
48       end
49     end
50     text = fixup_rss(text)
51     rss_list << RSS::Parser.parse(text, false)
52   end
53   rss_list
54 end
fixup_rss(text) click to toggle source

euc-jp may fail to parse

   # File lib/td2planet/fetcher.rb
57 def fixup_rss(text)
58   text.sub(/\bencoding="euc-jp"/ni, 'encoding="euc-jp-ms"')
59 end