class LogStash::Inputs::Signalsciences
Fetch Signal Sciences request data.
Public Instance Methods
fetch(queue)
click to toggle source
# File lib/logstash/inputs/signalsciences.rb, line 96 def fetch(queue) if @auth_mode == "password" begin response = @http.request(@login) @logger.warn("login response: #{response.code}") rescue @logger.warn("Could not reach API endpoint to login!") return false end json = JSON.parse(response.body) if json.has_key? "message" # failed to login @logger.warn("login: #{json['message']}") return false end bearer_token = json['token'] end # Both the from and until parameters must fall on full minute boundaries, # see https://docs.signalsciences.net/faq/extract-your-data/. t = Time.now.utc.strftime("%Y-%m-%d %H:%M:0") dt = DateTime.parse(t) timestamp_until = dt.to_time.to_i - 300 # now - 5 minutes timestamp_from = (timestamp_until - @from) # @until - @from if @debug hfrom = Time.at(timestamp_from).to_datetime huntil = Time.at(timestamp_until).to_datetime @logger.info("From #{hfrom} Until #{huntil}") end # Set up iniital get request and initial next_uri @logger.info("Requesting data: /api/v0/corps/#{@corp}/sites/#{@site}/feed/requests?from=#{timestamp_from}&until=#{timestamp_until}") get = Net::HTTP::Get.new("/api/v0/corps/#{@corp}/sites/#{@site}/feed/requests?from=#{timestamp_from}&until=#{timestamp_until}") next_uri = "not empty on first pass" # Loop through results until next_uri is empty. while !next_uri.empty? if @auth_mode == "password" get["Authorization"] = "Bearer #{bearer_token}" else get["x-api-user"] = @email get["x-api-token"] = @token end get['User-Agent'] = "logstash-signalsciences/#{@version}" begin response = @http.request(get) rescue @logger.warn("Could not reach API endpoint to retreive reqeusts feed!") return false end if response.code == "524" @logger.warn("524 - Origin Timeout!") @logger.info("Another attempt will be made later.") return false end if response.code == "429" @logger.warn("429 - Too Many Requests!") @logger.info("API request throttling as been triggered, another attempt will be made later. Contact support if this error continues.") return false end if response.code == "404" @logger.warn("404 - Not Found!") return false end if response.code == "401" @logger.warn("401 - Unauthorized!") return false end json = JSON.parse(response.body) #check for message, error, e.g. missing query string parameter if json.has_key? "message" # some error occured, report it. @logger.warn("Error accessing API (auth mode: #{@auth_mode}), status code: #{response.code} with message: #{json['message']}") return false end # log json payloads json['data'].each do |payload| # explode headersIn out to headerIn entries temp = {} begin payload['headersIn'].each { |k,v| temp[k] = v } payload["headerIn"] = temp rescue NoMethodError if @debug @logger.debug("payload['headersIn'] is empty for id #{payload['id']}, skipping append.") end end # explode headersOut out to headerOut entries temp = {} begin payload['headersOut'].each { |k,v| temp[k] = v } payload["headerOut"] = temp rescue NoMethodError if @debug @logger.info("payload['headersOut'] is empty for id #{payload['id']}, skipping append.") end end # explode tags out to tag entries temp = {} payload['tags'].each do |x| temp[x['type']] = x end payload['tag'] = temp # add the event event = LogStash::Event.new("message" => payload, "host" => @host) decorate(event) queue << event end # get the next uri value next_uri = json['next']['uri'] if @debug logger.info("Next URI: #{next_uri}") end # continue retreiving next_uri if it exists if !next_uri.empty? get = Net::HTTP::Get.new(next_uri) end end return true end
register()
click to toggle source
# File lib/logstash/inputs/signalsciences.rb, line 37 def register @host = Socket.gethostname @http = Net::HTTP.new('dashboard.signalsciences.net', 443) @http.set_debug_output($stdout) if @debug @login = Net::HTTP::Post.new("/api/v0/auth") @auth_mode = "password" @get = Net::HTTP::Get.new("/api/v0/corps/#{@corp}/sites/#{@site}/feed/requests") @http.use_ssl = true # determine authentication mode if @token != "" @auth_mode = "token" end # check if from value is less than 1 min if @from < 60 @logger.warn("from value is less than 1 min, increasing from value to 1 minute.") @from = 60 end # check if from value is greater than 24 hours if @from > 86400 @logger.warn("from value is greater than 24 hours, reducing from value to 24 hours.") @from = 86400 end # set interval to value of from @from minus five minutes @interval = @from # set version for UA string @version = "1.0.0" @logger.info("Fetching Signal Sciences request data every #{@interval / 60} minutes.") end
run(queue)
click to toggle source
# File lib/logstash/inputs/signalsciences.rb, line 72 def run(queue) # we can abort the loop if stop? becomes true while !stop? if @auth_mode == "password" @login['User-Agent'] = "logstash-signalsciences/#{@version}" @login.body = URI.encode_www_form({"email" => @email, "password" => @password}) end if fetch(queue) @logger.info("Requests feed retreived successfully.") else @logger.warn("Problem retreiving request!") end # because the sleep interval can be big, when shutdown happens # we want to be able to abort the sleep # Stud.stoppable_sleep will frequently evaluate the given block # and abort the sleep(@interval) if the return value is true #Stud.stoppable_sleep(@interval) { stop? } @logger.info("Sleep #{@interval}") Stud.stoppable_sleep(@interval) { stop? } end # loop end
stop()
click to toggle source
# File lib/logstash/inputs/signalsciences.rb, line 239 def stop # nothing to do in this case so it is not necessary to define stop # examples of common "stop" tasks: # * close sockets (unblocking blocking reads/accepts) # * cleanup temporary files # * terminate spawned threads end