class Ebooks::Archive

Attributes

tweets[R]

Public Class Methods

new(username, path=nil, client=nil) click to toggle source
# File lib/bot_twitter_ebooks/archive.rb, line 42
def initialize(username, path=nil, client=nil)
  @username = username
  @path = path || "corpus/#{username}.json"

  if File.directory?(@path)
    @path = File.join(@path, "#{username}.json")
  end

  @client = client || make_client

  if (File.exists?(@path) && !File.zero?(@path))
    @filetext = File.read(@path, :encoding => 'utf-8')
    @tweets = JSON.parse(@filetext, symbolize_names: true)
    log "Currently #{@tweets.length} tweets for #{@username}"
  else
    @tweets.nil?
    log "New archive for @#{username} at #{@path}"
  end
end

Public Instance Methods

make_client() click to toggle source
# File lib/bot_twitter_ebooks/archive.rb, line 13
def make_client
  if File.exists?(CONFIG_PATH)
    @config = JSON.parse(File.read(CONFIG_PATH), symbolize_names: true)
  else
    @config = {}

    puts "As Twitter no longer allows anonymous API access, you'll need to enter the auth details of any account to use for archiving. These will be stored in #{CONFIG_PATH} if you need to change them later."
    print "Consumer key: "
    @config[:consumer_key] = STDIN.gets.chomp
    print "Consumer secret: "
    @config[:consumer_secret] = STDIN.gets.chomp
    print "Access token: "
    @config[:oauth_token] = STDIN.gets.chomp
    print "Access secret: "
    @config[:oauth_token_secret] = STDIN.gets.chomp

    File.open(CONFIG_PATH, 'w') do |f|
      f.write(JSON.pretty_generate(@config))
    end
  end

  Twitter::REST::Client.new do |config|
    config.consumer_key = @config[:consumer_key]
    config.consumer_secret = @config[:consumer_secret]
    config.access_token = @config[:oauth_token]
    config.access_token_secret = @config[:oauth_token_secret]
  end
end
sync() click to toggle source
# File lib/bot_twitter_ebooks/archive.rb, line 62
def sync
  # We use this structure to ensure that
  # a) if there's an issue opening the file, we error out before download
  # b) if there's an issue during download we restore the original
  File.open(@path, 'w') do |file|
    begin
      sync_to(file)
    rescue Exception
      file.seek(0)
      file.write(@filetext)
      raise
    end
  end
end
sync_to(file) click to toggle source
# File lib/bot_twitter_ebooks/archive.rb, line 77
def sync_to(file)
  retries = 0
  tweets = []
  max_id = nil

  opts = {
    count: 200,
    #include_rts: false,
    tweet_mode: "extended",
    trim_user: true
  }

  opts[:since_id] = @tweets[0][:id] unless @tweets.nil?

  loop do
    opts[:max_id] = max_id unless max_id.nil?
    begin
      new = @client.user_timeline(@username, opts)
    rescue Twitter::Error::TooManyRequests
      log "Rate limit exceeded. Waiting for 15 mins before retry."
      sleep 60*15
      retry
    end
    break if new.length <= 1
    tweets += new
    log "Received #{tweets.length} new tweets"
    max_id = new.last.id - 1
  end

  if tweets.length == 0
    log "No new tweets"
  else
    @tweets ||= []
    @tweets = tweets.map(&:attrs).each { |tw|
      tw.replace({ text: tw[:full_text], id: tw[:id] })
    } + @tweets
  end
  file.write(JSON.pretty_generate(@tweets))
end