class WebPageParser::TheInterceptPageParserV1

TheInterceptPageParserV1 parses “The Intercept” web pages using html parsing.

Public Instance Methods

content() click to toggle source
# File lib/web-page-parser/parsers/the_intercept_page_parser.rb, line 43
def content
  return @content if @content
  story_body = html_doc.css('div.PostContent div p, article div.ti-body p').collect do |p|
    p.text.strip.gsub(160.chr(Encoding::UTF_8), ' ') # convert   to actual space
  end
  @content = story_body.select { |p| !p.empty? }
end
date() click to toggle source
# File lib/web-page-parser/parsers/the_intercept_page_parser.rb, line 51
def date
  return @date if @date
  if date_meta = html_doc.at_css('meta[property="article:published_time"]')
    date_string = date_meta['content'].scan(/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\+[0-9]{2}:[0-9]{2}/).first
    @date = DateTime.parse(date_string) rescue nil
  end
  return @date if @date
  if date_span = html_doc.css('span.PostByline-date')
    date_string = date_span.text.strip
    @date = DateTime.parse(date_string) rescue nil
  end
  return @date if @date
  # failing that, get it from the url
  @date = DateTime.parse(url.scan(/[0-9]{4}\/[0-9]{2}\/[0-9]{2}/).first.to_s) rescue nil
end
guid_from_url() click to toggle source

WashPo articles have a guid in the url (as of Jan 2014, a uuid)

# File lib/web-page-parser/parsers/the_intercept_page_parser.rb, line 20
def guid_from_url
  # get the last large number from the url, if there is one
  url.to_s.scan(/https:\/\/firstlook.org\/theintercept\/[0-9]{4}\/[0-9]{2}\/[0-9]{2}\/[a-z0-9-]+/).last
end
html_doc() click to toggle source
# File lib/web-page-parser/parsers/the_intercept_page_parser.rb, line 25
def html_doc
  @html_document ||= Nokogiri::HTML(page)
end
title() click to toggle source
# File lib/web-page-parser/parsers/the_intercept_page_parser.rb, line 29
def title
  return @title if @title
  title_meta = html_doc.at_css('meta[property="og:title"]')
  title = nil
  if title_meta
    title = title_meta['content'].to_s.strip
  end
  if title.nil?
    title = html_doc.css('head title').text.strip
  end
  title = title.gsub(/- The Intercept$/,'')
  @title = title.strip
end