class GitTrend::Scraper

Constants

BASE_HOST
BASE_URL

Public Class Methods

new() click to toggle source
# File lib/git_trend/scraper.rb, line 9
def initialize
  @agent = Mechanize.new
  @agent.user_agent = "git-trend #{VERSION}"
  proxy = URI.parse(ENV["http_proxy"]) if ENV["http_proxy"]
  @agent.set_proxy(proxy.host, proxy.port, proxy.user, proxy.password) if proxy
end

Public Instance Methods

get(language = nil, since = nil, number = nil) click to toggle source
# File lib/git_trend/scraper.rb, line 16
def get(language = nil, since = nil, number = nil)
  page = @agent.get(generate_url(language, since))
  projects = generate_project(page)
  number ? projects[0...number] : projects
end
languages() click to toggle source
# File lib/git_trend/scraper.rb, line 22
def languages
  page = @agent.get(BASE_URL)
  page.search("#select-menu-language .select-menu-list .select-menu-item-text").inject([]) do |languages, content|
    language = content.text.strip
    languages << language if language
  end
end

Private Instance Methods

comma_to_i(obj) click to toggle source
# File lib/git_trend/scraper.rb, line 66
def comma_to_i(obj)
  obj.to_s.delete(",").to_i
end
convert_url_param_since(since) click to toggle source
# File lib/git_trend/scraper.rb, line 42
def convert_url_param_since(since)
  return unless since
  case since.to_sym
  when :d, :day,   :daily   then "daily"
  when :w, :week,  :weekly  then "weekly"
  when :m, :month, :monthly then "monthly"
  else ""
  end
end
generate_project(page) click to toggle source
# File lib/git_trend/scraper.rb, line 52
def generate_project(page)
  page.search(".Box-row").map do |content|
    icon_area = content.search(".f6.color-fg-muted.mt-2")
    Project.new(
      name: content.search("h1 a").attr("href").to_s.sub(/\A\//, ""),
      description: content.search(".col-9.color-fg-muted.my-1.pr-4").text.strip,
      lang: content.search('span[itemprop="programmingLanguage"]').text.strip,
      all_star_count: comma_to_i(icon_area.search("a:has(svg.octicon-star)").text.strip),
      fork_count: comma_to_i(icon_area.search("a:has(svg.octicon-repo-forked)").text.strip),
      star_count: comma_to_i(icon_area.search(".float-sm-right").text)
    )
  end
end
generate_url(language, since) click to toggle source
# File lib/git_trend/scraper.rb, line 31
def generate_url(language, since)
  url = BASE_URL.dup
  url << "/#{language}" if language
  uri = Addressable::URI.parse(url)
  since = convert_url_param_since(since)
  if since
    uri.query_values = { since: since }.delete_if { |_k, v| v.nil? }
  end
  uri.to_s
end