A code which I am writing is heavily numerical in nature, but ultimately it calculates a value for each point of a 4D array according to the values within multiple separate 4D arrays. Consequently, now that I have a working proof-of-concept in Julia, I am looking to optimise it and make best use of available computational resources. To this end, I am attempting to parallelise the code by utilising pmap(), which the parallel computing documentation indicates
is designed for the case where each function call does a large amount of work.
Following the documentation and various forum posts, from both Discourse and SO, I have put together a code which calls pmap() and passes calculated values back into a 4D SharedArray. For n=1 workers, this runs without error and the results from test cases are correct. However, when attempting to upscale the number of workers (n>1), I receive a BoundsError on worker 2, apparently attempting to access elements of a 0x0x0x0 Array. My suspicion is that I have misused/am missing @everywheres, but perhaps someone could shed some light on this problem? A heavily abridged MWE of my code follows.
The main file:
#pmapdemo.jl
using Distributed
#addprocs(length(Sys.cpu_info())) # uncomment this line for error
@everywhere include(joinpath(@__DIR__, "pmapdemo2.jl"))
function main()
source = Dict{String, Any}("t"=>zeros(5),
"x"=>zeros(5,6),
"y"=>zeros(5,3),
"z"=>zeros(5,3))
iterset = Dict{String, Any}("t"=>source["t"],
"x"=>source["x"],
"y"=>fill(2, size(source["t"])[1], 1),
"z"=>fill(2, size(source["t"])[1], 1))
data = Dict{String, Any}()
MyMod.initialisearray!(data, iterset)
MyMod.calcarray!(data, iterset, source)
@show data
end
main()
The functionality file:
#pmapdemo2.jl
module MyMod
using Distributed
@everywhere using SharedArrays
function initialisearray!(data, fieldset)
zerofield::SharedArray{Float64, 4} = zeros(size(fieldset["t"])[1],
size(fieldset["x"])[2],
size(fieldset["y"])[2],
size(fieldset["z"])[2])
data["field"] = deepcopy(zerofield)
end
function calcpoint!((data, source, a, b, c, d))
data["field"][a,b,c,d] = rand()
end
function calcarray!(data, iterset, source)
for a in eachindex(iterset["t"])
b = eachindex(iterset["x"][a,:])
c = eachindex(iterset["y"][a,:])
d = eachindex(iterset["z"][a,:])
pmap(calcpoint!, Iterators.product(Iterators.repeated(data,1), Iterators.repeated(source,1), Iterators.repeated(a,1), b, c, d))
end
end
end
The output:
ERROR: LoadError: On worker 2:
BoundsError: attempt to access 0×0×0×0 Array{Float64,4} at index [1]
setindex! at ./array.jl:767 [inlined]
setindex! at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/SharedArrays/src/SharedArrays.jl:500 [inlined]
_setindex! at ./abstractarray.jl:1043
setindex! at ./abstractarray.jl:1020
calcpoint! at /home/dave/pmapdemo2.jl:25
#112 at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/process_messages.jl:269
run_work_thunk at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/process_messages.jl:56
macro expansion at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/process_messages.jl:269 [inlined]
#111 at ./task.jl:259
Stacktrace:
[1] (::getfield(Base, Symbol("##696#698")))(::Task) at ./asyncmap.jl:178
[2] foreach(::getfield(Base, Symbol("##696#698")), ::Array{Any,1}) at ./abstractarray.jl:1866
[3] maptwice(::Function, ::Channel{Any}, ::Array{Any,1}, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at ./asyncmap.jl:178
[4] #async_usemap#681 at ./asyncmap.jl:154 [inlined]
[5] #async_usemap at ./none:0 [inlined]
[6] #asyncmap#680 at ./asyncmap.jl:81 [inlined]
[7] #asyncmap at ./none:0 [inlined]
[8] #pmap#213(::Bool, ::Int64, ::Nothing, ::Array{Any,1}, ::Nothing, ::Function, ::Function, ::WorkerPool, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/pmap.jl:126
[9] pmap(::Function, ::WorkerPool, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/pmap.jl:101
[10] #pmap#223(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::Function, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/pmap.jl:156
[11] pmap(::Function, ::Base.Iterators.ProductIterator{Tuple{Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Dict{String,Any}}},Base.Iterators.Take{Base.Iterators.Repeated{Int64}},Base.OneTo{Int64},Base.OneTo{Int64},Base.OneTo{Int64}}}) at /build/julia/src/julia-1.1.1/usr/share/julia/stdlib/v1.1/Distributed/src/pmap.jl:156
[12] calcarray!(::Dict{String,Any}, ::Dict{String,Any}, ::Dict{String,Any}) at /home/dave/pmapdemo2.jl:20
[13] main() at /home/dave/pmapdemo.jl:19
[14] top-level scope at none:0
in expression starting at /home/dave/pmapdemo.jl:23