class TomosiaIcon8Crawl::CrawlIcon8
Public Class Methods
crawl(keyword = nil, path = ".", max = nil)
click to toggle source
main
# File lib/tomosia_icon8_crawl.rb, line 133 def self.crawl(keyword = nil, path = ".", max = nil) begin images = [] des = "" json(keyword, max) @responses['icons'].each_with_index do |item, index| src = "https://img.icons8.com/#{item['platform']}/2x/#{item['commonName']}.png" # add image images.push(src) end multi_download_image(path, images) save_file_excel(path, @data) rescue Exception => e p "--Runtime error--" p e end end
download_image(path, img)
click to toggle source
download image
# File lib/tomosia_icon8_crawl.rb, line 86 def self.download_image(path, img) timeout = 0 begin open(img) do |image| File.open(path, 'wb') do |file| file.write(image.read) @size = image.size end end rescue if timeout < 5 timeout += 1 p "Retry download image" retry end end end
json(key, max = nil)
click to toggle source
get data from website
# File lib/tomosia_icon8_crawl.rb, line 8 def self.json(key, max = nil) if key == nil p "No data!" else if max == nil urlc = 'https://search.icons8.com/api/iconsets/v5/search?term=' + key + '&amount=1' page = HTTParty.get(urlc) res = page.parsed_response countAll = res['parameters']['countAll'] url = 'https://search.icons8.com/api/iconsets/v5/search?term=' + key + '&amount=' + countAll.to_s else url = 'https://search.icons8.com/api/iconsets/v5/search?term=' + key + '&amount=' + max end end page_data = HTTParty.get(url) @responses = page_data.parsed_response end
multi_download_image(path, imgs)
click to toggle source
multi download image
# File lib/tomosia_icon8_crawl.rb, line 105 def self.multi_download_image(path, imgs) p "Start download image" print "Loadding" begin threads = [] @data = [] imgs.each_with_index do |img, index| # p index title = File.basename(img, '.png') des = path. + "/" + index.to_s + "-" + title + ".png" ext = File.extname(img).delete('.') threads << Thread.new do download_image(des, img) row = {"index" => index, "name" => title, "url" => img, "size" => @size, "extension" => ext} @data.push(row) print "." end end threads.each{ |t| t.join } rescue Exception => e p "no data" p e end print "\nDownload success!\n" end
save_file_excel( path, data = {})
click to toggle source
save file to excel
# File lib/tomosia_icon8_crawl.rb, line 35 def self.save_file_excel( path, data = {}) p "Write excel" begin des = path + '/export.xls' workbook = WriteExcel.new(des) format = workbook.add_format format.set_bold() format.set_align('center') data_col = workbook.add_format data_col.set_align('center') format_url = workbook.add_format format_url.set_color('blue') format_url.set_align('center') worksheet = workbook.add_worksheet worksheet.write_string(0, 0, 'STT', format) worksheet.write_string(0, 1, 'NAME', format) worksheet.write_string(0, 2, 'URL', format) worksheet.write_string(0, 3, 'SIZE(byte)', format) worksheet.write_string(0, 4, 'EXTENSION', format) multi = [] data.each_with_index do |row, index| i = index + 1 # p i # p row multi << Thread.new do row.each do |key, value| # p key worksheet.write_string(i, 0, row['index'], data_col) worksheet.write_string(i, 1, row['name'], data_col) worksheet.write_url(i, 2, row['url'], format_url) worksheet.write_string(i, 3, row['size'], data_col) worksheet.write_string(i, 4, row['extension'], data_col) end end end multi.each{ |m| m.join } workbook.close rescue Exception => e p "Can't saved file" p e end p "Write excel susscess!" end
save_file_txt(index, name, url, size, extension)
click to toggle source
save file to txt
# File lib/tomosia_icon8_crawl.rb, line 28 def self.save_file_txt(index, name, url, size, extension) File.open("log_image.txt", "a+") do |f| f.write("#{index}. name: #{name} | url: #{url} | size: #{size}Kb | extension: #{extension} \n") end end