class JunglePath::DBAccess::IO::ChunkedFileReader
Attributes
chunk_size[R]
data_file[R]
done[R]
line[R]
Public Class Methods
new(data_file)
click to toggle source
# File lib/jungle_path/db_access/io/chunked_file_reader.rb, line 9 def initialize data_file @data_file = data_file @start_at = 1 @chunk_size = 1000 @done = false @line = nil end
Public Instance Methods
each() { |line| ... }
click to toggle source
# File lib/jungle_path/db_access/io/chunked_file_reader.rb, line 36 def each @line = nil pointer = 0 yield_size = 0 File.open(@data_file, 'r') do |file| puts "reading: #{data_file}." while line = file.gets pointer += 1 if pointer == @start_at puts "starting chunk at #{pointer}." end puts " at line number #{pointer}." if pointer % 10000 == 0 if pointer >= @start_at and yield_size < @chunk_size @line = line yield_size += 1 yield line end end if yield_size == 0 @done = true end end end
was_bad_chunk()
click to toggle source
# File lib/jungle_path/db_access/io/chunked_file_reader.rb, line 17 def was_bad_chunk if @chunk_size == 1 @start_at = @start_at + 1 else @chunk_size = @chunk_size / 2 if @chunk_size < 1 @chunk_size = 1 end end end
was_good_chunk()
click to toggle source
# File lib/jungle_path/db_access/io/chunked_file_reader.rb, line 28 def was_good_chunk @start_at = @start_at + @chunk_size @chunk_size = @chunk_size * 2 if @chunk_size > 1000000 @chunk_size = 1000000 end end