Help implementing parallelism with readbytes!

General Usage Performance
, ,
4

You can pass a reference to a buffer if you want, something like:

c=Channel{Vector{UInt8}}(1) #replace with the actual type
function read_data!(c, f)
    # create two buffers so one can be written to and the other processed at the same time
    buffer1=zeros(UInt8, buffer_size)
    buffer2=similar(buffer1)
    buffer=buffer1
    open(f, 'r') do file
        while !eof(file)
             readbytes!(buffer, file) # read a chunk
             put!(c, buffer)
             buffer = (buffer==buffer1) ? buffer2 : buffer1 # switch ref to buffer
        end
    end
end
function process_data!(c)
    while isopen(c)
        data=take!(c)
        # process the data
    end
end
processing_task = Threads.@spawn process_data!(c)
read_data!(c, "filename.dat") # can read on main thread
wait(processing_task)
2 Likes