class Rack::Prerendercloud
Public Class Methods
header_whitelist()
click to toggle source
# File lib/prerendercloud.rb, line 8 def header_whitelist # preserve (and send to client) these headers from service.prerender.cloud # which originally came from the origin server return [ "vary", "content-type", "cache-control", "strict-transport-security", "content-security-policy", "public-key-pins", "x-frame-options", "x-xss-protection", "x-content-type-options", "location" ]; end
new(app, options={})
click to toggle source
# File lib/prerendercloud.rb, line 35 def initialize(app, options={}) # googlebot, yahoo, and bingbot are not in this list because # we support _escaped_fragment_ and want to ensure people aren't # penalized for cloaking. @crawler_user_agents = [ 'googlebot', 'yahoo', 'bingbot', 'baiduspider', 'facebookexternalhit', 'twitterbot', 'rogerbot', 'linkedinbot', 'embedly', 'bufferbot', 'quora link preview', 'showyoubot', 'outbrain', 'pinterest/0.', 'developers.google.com/+/web/snippet', 'www.google.com/webmasters/tools/richsnippets', 'slackbot', 'vkShare', 'W3C_Validator', 'redditbot', 'Applebot', 'WhatsApp', 'flipboard', 'tumblr', 'bitlybot', 'SkypeUriPreview', 'nuzzel', 'Discordbot', 'Google Page Speed', 'Qwantify' ] @options = options @options[:whitelist] = [@options[:whitelist]] if @options[:whitelist].is_a? String @options[:blacklist] = [@options[:blacklist]] if @options[:blacklist].is_a? String @crawler_user_agents = @options[:crawler_user_agents] if @options[:crawler_user_agents] @app = app end
normalize_headers(headers)
click to toggle source
# File lib/prerendercloud.rb, line 25 def normalize_headers(headers) filtered = headers.select { |key, value| header_whitelist.any? { |h| key.to_s.match(/#{h}/i) } } mapped = filtered.map do |k, v| [k, if v.is_a? Array then v.join("\n") else v end] end Utils::HeaderHash.new Hash[mapped] end
Public Instance Methods
after_render(env, response)
click to toggle source
# File lib/prerendercloud.rb, line 259 def after_render(env, response) return true unless @options[:after_render] @options[:after_render].call(env, response) end
before_render(env)
click to toggle source
# File lib/prerendercloud.rb, line 244 def before_render(env) return nil unless @options[:before_render] cached_render = @options[:before_render].call(env) if cached_render && cached_render.is_a?(String) Rack::Response.new(cached_render, 200, { 'Content-Type' => 'text/html; charset=utf-8' }) elsif cached_render && cached_render.is_a?(Rack::Response) cached_render else nil end end
build_api_url(env)
click to toggle source
# File lib/prerendercloud.rb, line 201 def build_api_url(env) new_env = env if env["CF-VISITOR"] match = /"scheme":"(http|https)"/.match(env['CF-VISITOR']) new_env["HTTPS"] = true and new_env["rack.url_scheme"] = "https" and new_env["SERVER_PORT"] = 443 if (match && match[1] == "https") new_env["HTTPS"] = false and new_env["rack.url_scheme"] = "http" and new_env["SERVER_PORT"] = 80 if (match && match[1] == "http") end if env["X-FORWARDED-PROTO"] new_env["HTTPS"] = true and new_env["rack.url_scheme"] = "https" and new_env["SERVER_PORT"] = 443 if env["X-FORWARDED-PROTO"].split(',')[0] == "https" new_env["HTTPS"] = false and new_env["rack.url_scheme"] = "http" and new_env["SERVER_PORT"] = 80 if env["X-FORWARDED-PROTO"].split(',')[0] == "http" end if @options[:protocol] new_env["HTTPS"] = true and new_env["rack.url_scheme"] = "https" and new_env["SERVER_PORT"] = 443 if @options[:protocol] == "https" new_env["HTTPS"] = false and new_env["rack.url_scheme"] = "http" and new_env["SERVER_PORT"] = 80 if @options[:protocol] == "http" end url = Rack::Request.new(new_env).url puts "Requesting prerendered page for #{url}" prerender_url = get_prerender_service_url() forward_slash = prerender_url[-1, 1] == '/' ? '' : '/' "#{prerender_url}#{forward_slash}#{url}" end
build_rack_response_from_prerender(prerendered_response)
click to toggle source
# File lib/prerendercloud.rb, line 232 def build_rack_response_from_prerender(prerendered_response) headers = (prerendered_response.respond_to?(:headers) && prerendered_response.headers) || self.class.normalize_headers(prerendered_response.to_hash) response = Rack::Response.new(prerendered_response.body, prerendered_response.code, headers) @options[:build_rack_response_from_prerender].call(response, prerendered_response) if @options[:build_rack_response_from_prerender] response end
call(env)
click to toggle source
# File lib/prerendercloud.rb, line 80 def call(env) if should_show_prerendered_page(env) cached_response = before_render(env) if cached_response return cached_response.finish end prerendered_response = get_prerendered_page_response(env) if prerendered_response response = build_rack_response_from_prerender(prerendered_response) after_render(env, prerendered_response) return response.finish end end @app.call(env) end
get_prerender_service_url()
click to toggle source
# File lib/prerendercloud.rb, line 227 def get_prerender_service_url @options[:prerender_service_url] || ENV['PRERENDER_SERVICE_URL'] || 'http://service.prerender.cloud/' end
get_prerendered_page_response(env)
click to toggle source
# File lib/prerendercloud.rb, line 175 def get_prerendered_page_response(env) begin url = URI.parse(build_api_url(env)) headers = { 'User-Agent' => env['HTTP_USER_AGENT'], 'Accept-Encoding' => 'gzip' } headers['X-Prerender-Token'] = ENV['PRERENDER_TOKEN'] if ENV['PRERENDER_TOKEN'] headers['X-Prerender-Token'] = @options[:prerender_token] if @options[:prerender_token] req = Net::HTTP::Get.new(url.request_uri, headers) http = Net::HTTP.new(url.host, url.port) http.use_ssl = true if url.scheme == 'https' response = http.request(req) if response['Content-Encoding'] == 'gzip' response.body = ActiveSupport::Gzip.decompress(response.body) response['Content-Length'] = response.body.length response.delete('Content-Encoding') end response rescue => e puts "Error: #{e.class.name}" nil end end
prerenderable_extension(fullpath)
click to toggle source
# File lib/prerendercloud.rb, line 157 def prerenderable_extension(fullpath) path = URI.parse(fullpath).path; # doesn't detect index.whatever.html (multiple dots) hasHtmlOrNoExtension = !!path.match(/^(([^.]|\.html?)+)$/); return true if hasHtmlOrNoExtension # hack to handle basenames with multiple dots: index.whatever.html endsInHtml = !!path.match(/.html?$/); return true if endsInHtml return false; end
should_show_prerendered_page(env)
click to toggle source
# File lib/prerendercloud.rb, line 102 def should_show_prerendered_page(env) user_agent = env['HTTP_USER_AGENT'] buffer_agent = env['HTTP_X_BUFFERBOT'] is_requesting_prerendered_page = false return false if !user_agent return false if env['REQUEST_METHOD'] != 'GET' return false if user_agent.match(/prerendercloud/i); request = Rack::Request.new(env) return false if !prerenderable_extension(request.fullpath); #if it is a bot...show prerendered page if (@options[:bots_only]) is_requesting_prerendered_page = true if @crawler_user_agents.any? { |crawler_user_agent| user_agent.downcase.include?(crawler_user_agent.downcase) } #if it is BufferBot...show prerendered page is_requesting_prerendered_page = true if buffer_agent else is_requesting_prerendered_page = true end is_requesting_prerendered_page = true if Rack::Utils.parse_query(request.query_string).has_key?('_escaped_fragment_') # if whitelist exists and path is not whitelisted...don't prerender return false if @options[:whitelist].is_a?(Array) && @options[:whitelist].all? do |whitelisted| if whitelisted.is_a?(Regexp) !whitelisted.match(request.fullpath) else whitelisted != request.fullpath; end end # if blacklist exists and path is blacklisted(url or referer)...don't prerender if @options[:blacklist].is_a?(Array) && @options[:blacklist].any? do |blacklisted| blacklistedUrl = false blacklistedReferer = false if blacklisted.is_a?(Regexp) blacklistedUrl = !!blacklisted.match(request.fullpath) blacklistedReferer = !!blacklisted.match(request.referer) if request.referer else blacklistedUrl = blacklisted == request.fullpath blacklistedReferer = request.referer && blacklisted == request.referer end blacklistedUrl || blacklistedReferer end return false end return is_requesting_prerendered_page end