Help implementing parallelism with readbytes!

General Usage Performance
, ,
2

I would create a single channel first, and then have two functions, one which continuous writes to the channel (reading the bytes) and another which processes.

c=Channel{ByteDataType}(1) #replace with the actual type
function read_data!(c, f
    open(f, 'r') do file
        while !eof(file)
             data=readbytes!(file) # read a chunk
             put!(c, data)
        end
    end
end
function process_data!(c)
    while isopen(c)
        data=take!(c)
        # process the data
    end
end
processing_task = Threads.@spawn process_data!(c)
read_data!(c, "filename.dat") # can read on main thread
wait(processing_task)

Sorry for the " pseudocodeness", I am on my phone and can’t run anything now. I hope it is clear nonetheless.

1 Like