class TomosiaIcon8Crawl::CrawlIcon8

Public Class Methods

crawl(keyword = nil, path = ".", max = nil) click to toggle source

main

# File lib/tomosia_icon8_crawl.rb, line 133
def self.crawl(keyword = nil, path = ".", max = nil)
  begin
      
    images = []
    des = ""
    json(keyword, max)
    @responses['icons'].each_with_index do |item, index| 

      src = "https://img.icons8.com/#{item['platform']}/2x/#{item['commonName']}.png"

      # add image
      images.push(src)
    end

    multi_download_image(path, images)
    save_file_excel(path, @data)
  rescue Exception => e
    p "--Runtime error--"
    p e
  end
end
download_image(path, img) click to toggle source

download image

# File lib/tomosia_icon8_crawl.rb, line 86
def self.download_image(path, img)
  timeout = 0
  begin 
    open(img) do |image|
    File.open(path, 'wb') do |file|
      file.write(image.read)
        @size = image.size
      end 
    end
  rescue
    if timeout < 5
      timeout += 1
      p "Retry download image"
      retry
    end
  end     
end
json(key, max = nil) click to toggle source

get data from website

# File lib/tomosia_icon8_crawl.rb, line 8
def self.json(key, max = nil)
  if key == nil
    p "No data!"
  else 
    if max == nil
      urlc = 'https://search.icons8.com/api/iconsets/v5/search?term=' + key + '&amount=1'
      page = HTTParty.get(urlc)
      res = page.parsed_response
      countAll = res['parameters']['countAll']
      url = 'https://search.icons8.com/api/iconsets/v5/search?term=' + key + '&amount=' + countAll.to_s
    else
       url = 'https://search.icons8.com/api/iconsets/v5/search?term=' + key + '&amount=' + max
    end
  end
    
  page_data = HTTParty.get(url)
  @responses = page_data.parsed_response  
end
multi_download_image(path, imgs) click to toggle source

multi download image

# File lib/tomosia_icon8_crawl.rb, line 105
def self.multi_download_image(path, imgs)
  p "Start download image"
  print "Loadding"
  begin
    threads = []
    @data = []

    imgs.each_with_index do |img, index|
      # p index
      title = File.basename(img, '.png')
      des = path. + "/" + index.to_s + "-" + title + ".png"
      ext = File.extname(img).delete('.')
      threads << Thread.new do 
        download_image(des, img)
        row = {"index" => index, "name" => title, "url" => img, "size" => @size, "extension" => ext}
        @data.push(row)
        print "."
      end
    end
    threads.each{ |t| t.join }  
  rescue Exception => e
    p "no data"
    p e
  end
  print "\nDownload success!\n"
end
save_file_excel( path, data = {}) click to toggle source

save file to excel

# File lib/tomosia_icon8_crawl.rb, line 35
def self.save_file_excel( path, data = {})
  p "Write excel"
  begin
    des = path + '/export.xls'
    workbook  = WriteExcel.new(des)
    format = workbook.add_format
    format.set_bold()
    format.set_align('center')

    data_col = workbook.add_format
    data_col.set_align('center')
    format_url = workbook.add_format
    format_url.set_color('blue')
    format_url.set_align('center')

    worksheet = workbook.add_worksheet
        
    worksheet.write_string(0, 0, 'STT', format)
    worksheet.write_string(0, 1, 'NAME', format)
    worksheet.write_string(0, 2, 'URL', format)
    worksheet.write_string(0, 3, 'SIZE(byte)', format)
    worksheet.write_string(0, 4, 'EXTENSION', format)

    multi = []
    data.each_with_index do |row, index|
      i = index + 1
      # p i
      # p row
      multi << Thread.new do
        row.each do |key, value|
          # p key

          worksheet.write_string(i, 0, row['index'], data_col)
          worksheet.write_string(i, 1, row['name'], data_col)
          worksheet.write_url(i, 2, row['url'], format_url)
          worksheet.write_string(i, 3, row['size'], data_col)
          worksheet.write_string(i, 4, row['extension'], data_col)
        end
      end
    end
    multi.each{ |m| m.join }

    workbook.close
  rescue Exception => e
    p "Can't saved file"
    p e
  end
  p "Write excel susscess!"
end
save_file_txt(index, name, url, size, extension) click to toggle source

save file to txt

# File lib/tomosia_icon8_crawl.rb, line 28
def self.save_file_txt(index, name, url, size, extension)   
  File.open("log_image.txt", "a+") do |f|
    f.write("#{index}. name: #{name} | url: #{url} | size: #{size}Kb | extension: #{extension} \n")
  end
end