class DevTo::Scraper
Public Instance Methods
get_page()
click to toggle source
# File lib/dev_to/scraper.rb, line 3 def get_page Nokogiri::HTML(open("https://dev.to/")) end
make_content(current_post)
click to toggle source
# File lib/dev_to/scraper.rb, line 13 def make_content(current_post) post_page = Nokogiri::HTML(open("#{current_post.url}")) current_post.add_post_attributes(scrape_content(post_page)) end
make_posts()
click to toggle source
# File lib/dev_to/scraper.rb, line 7 def make_posts posts.each do |post| DevTo::Post.new(scrape_posts(post)) end end
Private Instance Methods
posts()
click to toggle source
# File lib/dev_to/scraper.rb, line 19 def posts posts ||= get_page.css(".articles-list .substories .single-article") end
scrape_content(post_page)
click to toggle source
# File lib/dev_to/scraper.rb, line 34 def scrape_content(post_page) { :date => post_page.search("#main-title .published-at").text.strip, :content => post_page.search("body #article-body").text.strip } end
scrape_posts(post)
click to toggle source
# File lib/dev_to/scraper.rb, line 23 def scrape_posts(post) { :title => post.search(".content h3").text.strip.gsub(/[[:space:]]+/," "), :author => post.search("h4").text.strip, :tags => post.search(".tags .tag").collect { |tag| tag.text.strip}, :url => "https://dev.to" + post.search("a").last.attr("href"), :comments => post.search(".engagement-count-number").first.text.strip, :likes => post.search(".engagement-count-number").last.text.strip, } end