class TD2Planet::Fetcher
Public Class Methods
new(cache_dir, dry_run=false, user_agent="td2planet")
click to toggle source
# File lib/td2planet/fetcher.rb 12 def initialize(cache_dir, dry_run=false, user_agent="td2planet") 13 @cache_dir = Pathname.new(cache_dir) 14 unless @cache_dir.exist? 15 @cache_dir.mkdir 16 end 17 @dry_run = dry_run 18 @user_agent = user_agent 19 end
Public Instance Methods
fetch_all_rss(uris)
click to toggle source
# File lib/td2planet/fetcher.rb 21 def fetch_all_rss(uris) 22 rss_list = [] 23 uris.each do |uri| 24 cache_file = @cache_dir + ERB::Util.u(uri) 25 if @dry_run 26 puts "use cache: #{cache_file}" 27 text = cache_file.read 28 else 29 text = nil 30 begin 31 puts "fetch: #{uri}" 32 text = URI.parse(uri).read("User-Agent" => @user_agent) 33 rescue Timeout::Error 34 # fallback 35 puts "ERROR: timeout #{uri}" 36 text = cache_file.read 37 rescue Exception 38 puts "ERROR: #{$!} (#{$!.class}) on #{uri}" 39 next 40 else 41 if text.status[0] == '200' && /rss/ =~ text 42 cache_file.open('wb'){|f| f.write(text) } 43 else 44 # fallback 45 puts "ERROR: fetch failed #{uri} #{text.status}" 46 text = cache_file.read 47 end 48 end 49 end 50 text = fixup_rss(text) 51 rss_list << RSS::Parser.parse(text, false) 52 end 53 rss_list 54 end
fixup_rss(text)
click to toggle source
euc-jp may fail to parse
# File lib/td2planet/fetcher.rb 57 def fixup_rss(text) 58 text.sub(/\bencoding="euc-jp"/ni, 'encoding="euc-jp-ms"') 59 end