module Google::SafeBrowsingHelper
Public Instance Methods
can_request_full_hash?(hash_prefix)
click to toggle source
# File lib/rails/generators/google/helper/templates/safe_browsing_helper.rb, line 63 def can_request_full_hash? hash_prefix now = Time.now request = Google::SafeBrowsingFullHashRequest.where(prefix: hash_prefix).first if request.nil? true else if request.state == SafeBrowsingFullHashRequest::COMPLETED if request.requested_at < now - Google::CONFIG['full_length_hash_expires'] true else Rails.logger.warn "Full hash [#{hash_prefix}] requested successfully recently, skip this time." false end else attempts = request.attempts.nil?? 1 : request.attempts max_delay = Google::CONFIG['full_length_hash_backoff_delay_max'] delay = attempts * Google::CONFIG['full_length_hash_backoff_delay'] delay = max_delay if delay > max_delay if request.requested_at > now - delay Rails.logger.warn "Full hash [#{hash_prefix}] request in backoff mode. Wait time is #{delay} seconds" false else true end end end end
find_host_key_hits(prefixes)
click to toggle source
# File lib/rails/generators/google/helper/templates/safe_browsing_helper.rb, line 141 def find_host_key_hits prefixes ret = [] SafeBrowsingShavar.add_host_keys(prefixes).each do |add| if SafeBrowsingShavar.find_subs_for_add(add.chunk_num, add.host_key, add.prefix).empty? ret << add end end ret end
find_prefix_key_hits(host_keys, prefixes)
click to toggle source
# File lib/rails/generators/google/helper/templates/safe_browsing_helper.rb, line 151 def find_prefix_key_hits host_keys, prefixes ret = [] SafeBrowsingShavar.add_host_prefixes(host_keys, prefixes).each do |add| if SafeBrowsingShavar.find_subs_for_add(add.chunk_num, add.host_key, add.prefix).empty? ret << add end end ret end
full_hash_cache_hits(full_hashes)
click to toggle source
# File lib/rails/generators/google/helper/templates/safe_browsing_helper.rb, line 161 def full_hash_cache_hits full_hashes SafeBrowsingFullHash.includes(:list).where(value: full_hashes) .where('updated_at > ?', Time.now - Google::CONFIG['full_length_hash_expires']) end
get_full_hash_list(hash_prefix_arr)
click to toggle source
hash_prefix_arr: ['5b3583c0', 'b3e357a6']
# File lib/rails/generators/google/helper/templates/safe_browsing_helper.rb, line 20 def get_full_hash_list hash_prefix_arr ret_full_hashes = [] api = SafeBrowsingClient.new return ret_full_hashes if hash_prefix_arr.empty? begin full_hash_objs = api.full_hash(hash_prefix_arr) || {} save_full_hash_requests(hash_prefix_arr, SafeBrowsingFullHashRequest::COMPLETED) rescue Google::Error::NoContent => e Rails.logger.warn "NoContent Error for hash prefixes [#{hash_prefix_arr.join(', ')}]" save_full_hash_requests(hash_prefix_arr, SafeBrowsingFullHashRequest::COMPLETED) return ret_full_hashes rescue Exception => e Rails.logger.error "Error backtrace #{e.backtrace.join(%Q(\n))}" Rails.logger.warn "Error (#{e.inspect}) for hash prefixes [#{hash_prefix_arr.join(', ')}], continue..." save_full_hash_requests(hash_prefix_arr, e.message) return ret_full_hashes end full_hash_objs.keys.each do |list| full_hash_objs[list].keys.each do |add_chunk_num| full_hash_objs[list][add_chunk_num].each do |full_hash| unless (list_obj = SafeBrowsingList.list_by_name(list)).nil? Rails.logger.info "Updating full hash data with #{list}:#{add_chunk_num}:#{full_hash}" local = SafeBrowsingFullHash.where(value: full_hash).first if local local.touch else local = SafeBrowsingFullHash.create(value: full_hash, add_chunk_num: add_chunk_num, google_safe_browsing_list_id: list_obj.id) end ret_full_hashes << full_hash end end end end ret_full_hashes end
hits_to_category(url, full_hits)
click to toggle source
# File lib/rails/generators/google/helper/templates/safe_browsing_helper.rb, line 132 def hits_to_category url, full_hits (ret = {}).tap do ret[url] ||= [] full_hits.each do |hit| ret[url] << hit.list.name end end end
lookup_url(url)
click to toggle source
# File lib/rails/generators/google/helper/templates/safe_browsing_helper.rb, line 91 def lookup_url url canon_url = Google::UrlCanonicalizer.apply(url) hosts_set, urls_set = Google::UrlScramble.gen(canon_url) full_hits = [] unless hosts_set.empty? host_shas = hosts_set.map {|x| ShaUtil.sha256_hex(x, 8)} host_hits = find_host_key_hits(host_shas).map(&:host_key) if host_hits.empty? Rails.logger.info "No host key prefix found. Return." return hits_to_category(url, full_hits) else url_shas = urls_set.map {|x| ShaUtil.sha256_hex(x, 8)} prefix_hits = find_prefix_key_hits(host_shas, url_shas).map(&:prefix) candidate_prefixes = (host_hits + prefix_hits).uniq full_hash_expressions = urls_set.map {|x| ShaUtil.sha256_hex(x)}.select {|x| candidate_prefixes.include?(x.first(8))} full_hits = full_hash_cache_hits(full_hash_expressions) if full_hits.empty? warm_prefixes = candidate_prefixes.select {|x| !can_request_full_hash?(x)} candidate_prefixes -= warm_prefixes unless candidate_prefixes.empty? Rails.logger.info "Asking Google for full length hash #{candidate_prefixes}" get_full_hash_list(candidate_prefixes) full_hits = full_hash_cache_hits(full_hash_expressions) end end end end if full_hits.empty? Rails.logger.info "Return no hits..." else Rails.logger.info "Return hits #{full_hits.inspect}..." end hits_to_category(url, full_hits) end
save_full_hash_requests(prefixes, state, requested_at = Time.now)
click to toggle source
# File lib/rails/generators/google/helper/templates/safe_browsing_helper.rb, line 5 def save_full_hash_requests prefixes, state, requested_at = Time.now prefixes.each do |pre| obj = SafeBrowsingFullHashRequest.where(prefix: pre).first if obj.nil? obj = SafeBrowsingFullHashRequest.create(prefix: pre, requested_at: requested_at, state: state) else obj.update_attributes(requested_at: requested_at, state: state) end end end