class JunglePath::DBAccess::IO::ChunkedFileReader

Attributes

chunk_size[R]

yield up chunks of lines. treat entire chunk as good or bad. if chunk size is 1 line and it is bad move start_at forward by 1 (chunk size). if chunk was good move start_at forward by chunk size.

data_file[R]

yield up chunks of lines. treat entire chunk as good or bad. if chunk size is 1 line and it is bad move start_at forward by 1 (chunk size). if chunk was good move start_at forward by chunk size.

done[R]

yield up chunks of lines. treat entire chunk as good or bad. if chunk size is 1 line and it is bad move start_at forward by 1 (chunk size). if chunk was good move start_at forward by chunk size.

line[R]

yield up chunks of lines. treat entire chunk as good or bad. if chunk size is 1 line and it is bad move start_at forward by 1 (chunk size). if chunk was good move start_at forward by chunk size.

start_at[R]

yield up chunks of lines. treat entire chunk as good or bad. if chunk size is 1 line and it is bad move start_at forward by 1 (chunk size). if chunk was good move start_at forward by chunk size.

Public Class Methods

new(data_file) click to toggle source
# File lib/jungle_path/db_access/io/chunked_file_reader.rb, line 9
def initialize data_file
        @data_file = data_file
        @start_at = 1
        @chunk_size = 1000
        @done = false
        @line = nil
end

Public Instance Methods

each() { |line| ... } click to toggle source
# File lib/jungle_path/db_access/io/chunked_file_reader.rb, line 36
def each
        @line = nil
        pointer = 0
        yield_size = 0
        File.open(@data_file, 'r') do |file|
                puts "reading: #{data_file}."
                while line = file.gets
                        pointer += 1
                        if pointer == @start_at
                                puts "starting chunk at #{pointer}."
                        end
                        puts "    at line number #{pointer}." if pointer % 10000 == 0
                        if pointer >= @start_at and yield_size < @chunk_size
                                @line = line
                                yield_size += 1
                                yield line
                        end
                end
                if yield_size == 0
                        @done = true
                end
        end
end
was_bad_chunk() click to toggle source
# File lib/jungle_path/db_access/io/chunked_file_reader.rb, line 17
def was_bad_chunk
        if @chunk_size == 1
                @start_at = @start_at + 1
        else
                @chunk_size = @chunk_size / 2
                if @chunk_size < 1
                        @chunk_size = 1
                end
        end
end
was_good_chunk() click to toggle source
# File lib/jungle_path/db_access/io/chunked_file_reader.rb, line 28
def was_good_chunk
        @start_at = @start_at + @chunk_size
        @chunk_size = @chunk_size * 2
        if @chunk_size > 1000000
                @chunk_size = 1000000
        end
end