class Statement::Feed
Public Class Methods
batch(urls)
click to toggle source
# File lib/statement/feed.rb, line 11 def self.batch(urls) results = [] failures = [] hydra = Typhoeus::Hydra.new urls.each do |url| req = Typhoeus::Request.new(url) req.on_complete do |response| if response.success? doc = Nokogiri::XML(response.body) results << parse_atom(doc, url) if url == "http://larson.house.gov/index.php?option=com_ninjarsssyndicator&feed_id=1&format=raw" results << parse_rss(doc, url) else failures << url end end hydra.queue(req) end hydra.run [results.flatten, failures] end
date_from_rss_item(link)
click to toggle source
# File lib/statement/feed.rb, line 40 def self.date_from_rss_item(link) if !link.xpath('pubDate').text.empty? Date.parse(link.xpath('pubDate').text) elsif !link.xpath('pubdate').empty? Date.parse(link.xpath('pubdate').text) elsif link.xpath('link').text.include?("mikulski.senate.gov") and link.xpath('link').text.include?("-2014") Date.parse(link.xpath('link').text.split('/').last.split('-', -1).first(3).join('/').split('.cfm').first) else nil end end
from_rss(url)
click to toggle source
# File lib/statement/feed.rb, line 52 def self.from_rss(url) doc = open_rss(url) return unless doc if url == "http://larson.house.gov/index.php?option=com_ninjarsssyndicator&feed_id=1&format=raw" parse_atom(doc, url) else parse_rss(doc, url) end end
open_rss(url)
click to toggle source
# File lib/statement/feed.rb, line 32 def self.open_rss(url) begin Nokogiri::XML(open(url)) rescue nil end end
parse_atom(doc, url)
click to toggle source
# File lib/statement/feed.rb, line 74 def self.parse_atom(doc, url) links = (doc/:entry) return if links.empty? results = links.map do |link| { :source => url, :url => link.children[3]['href'], :title => link.children[1].text, :date => Date.parse(link.children[5].text), :domain => URI.parse(url).host } end end
parse_rss(doc, url)
click to toggle source
# File lib/statement/feed.rb, line 62 def self.parse_rss(doc, url) links = doc.xpath('//item') return if links.empty? results = links.map do |link| abs_link = Utils.absolute_link(url, link.xpath('link').text) abs_link = "http://www.burr.senate.gov/public/"+ link.xpath('link').text if url == 'http://www.burr.senate.gov/public/index.cfm?FuseAction=RSS.Feed' abs_link = link.xpath('link').text[37..-1] if url == "http://www.johanns.senate.gov/public/?a=RSS.Feed" { :source => url, :url => abs_link, :title => link.xpath('title').text, :date => date_from_rss_item(link), :domain => URI.parse(url).host } end Utils.remove_generic_urls!(results) end