Error solve JuMP models in parallel

Specific Domains Optimization (Mathematical)
1

Hi folks,
I’m new to Julia. I’m solving some JuMP models in parallel and I encountered the following error. The smallest example that I can come up with is as follows:

smallmodel.jl

using JuMP
using Ipopt
function generate_model()
	model = Model(solver=IpoptSolver())
	I = 1:3
	@variable(model, x[i in I; i <=2]>=1)
	@objective(model, Min, sum(x[i] for i in I if i <=2))
	return model
end


function psolve(m::JuMP.Model)
	temp = solve(m)
	d = Dict()
	d[:status] = temp
	d[:x] = getvalue(getindex(m, :x))
	return d
end

In the smallmodel.jl, I define variable x whose index has to be i in I; i <=2. And a psolve function.

Then I try to solve the model both in sequel and in parallel.

addprocs(2)

@everywhere include("smallmodel.jl")

model_array = []
for i = 1:3
	push!(model_array, generate_model())
end

for i = 1:3
	solve(model_array[i])
	println(getvalue(getindex(model_array[i], :x)))
end
results = pmap(psolve, model_array)
println(results[3][:x])

The solve in sequel works well. However, if I solve in parallel, I encountered the following error with the getindex function

ERROR: LoadError: On worker 2:

getindex at ./associative.jl:363
_map at /home/canl1/.julia/v0.6/JuMP/src/JuMPContainer.jl:162
getvalue at /home/canl1/.julia/v0.6/JuMP/src/JuMPContainer.jl:114
psolve at /home/canl1/work/hierarchy/large/3/smallmodel.jl:16
#106 at ./distributed/process_messages.jl:268 [inlined]
run_work_thunk at ./distributed/process_messages.jl:56
macro expansion at ./distributed/process_messages.jl:268 [inlined]
#105 at ./event.jl:73
Stacktrace:
 [1] #571 at ./asyncmap.jl:178 [inlined]
 [2] foreach(::Base.##571#573, ::Array{Any,1}) at ./abstractarray.jl:1733
 [3] maptwice(::Function, ::Channel{Any}, ::Array{Any,1}, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./asyncmap.jl:178
 [4] wrap_n_exec_twice(::Channel{Any}, ::Array{Any,1}, ::Base.Distributed.##204#207{WorkerPool}, ::Function, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./asyncmap.jl:154
 [5] #async_usemap#556(::Function, ::Void, ::Function, ::Base.Distributed.##188#190, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./asyncmap.jl:103
 [6] (::Base.#kw##async_usemap)(::Array{Any,1}, ::Base.#async_usemap, ::Function, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./<missing>:0
 [7] (::Base.#kw##asyncmap)(::Array{Any,1}, ::Base.#asyncmap, ::Function, ::Array{Any,1}) at ./<missing>:0
 [8] #pmap#203(::Bool, ::Int64, ::Void, ::Array{Any,1}, ::Void, ::Function, ::WorkerPool, ::Function, ::Array{Any,1}) at ./distributed/pmap.jl:126
 [9] pmap(::WorkerPool, ::Function, ::Array{Any,1}) at ./distributed/pmap.jl:101
 [10] #pmap#213(::Array{Any,1}, ::Function, ::Function, ::Array{Any,1}) at ./distributed/pmap.jl:156
 [11] pmap(::Function, ::Array{Any,1}) at ./distributed/pmap.jl:156
 [12] include_from_node1(::String) at ./loading.jl:576
 [13] include(::String) at ./sysimg.jl:14
 [14] process_options(::Base.JLOptions) at ./client.jl:305
 [15] _start() at ./client.jl:371fatal: error thrown and no exception handler available.
Base.AssertionError(msg="j isa Array{Variable}")
rec_backtrace at /buildworker/worker/package_linux64/build/src/stackwalk.c:84
record_backtrace at /buildworker/worker/package_linux64/build/src/task.c:245
jl_throw at /buildworker/worker/package_linux64/build/src/task.c:564
cont_str at /home/canl1/.julia/v0.6/JuMP/src/print.jl:417
unknown function (ip: 0x7fad186e5e9d)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1424 [inlined]
jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:51
show at /home/canl1/.julia/v0.6/JuMP/src/print.jl:397 [inlined]
showall at ./show.jl:1719
repr at ./strings/io.jl:146
unknown function (ip: 0x7fad186e15e2)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
showerror at ./replutil.jl:282
#476 at ./replutil.jl:213
unknown function (ip: 0x7fad186e13e2)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#with_output_color#512 at ./util.jl:400
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
with_output_color at ./util.jl:397
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#showerror#475 at ./replutil.jl:212
unknown function (ip: 0x7fad186e0919)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#showerror at ./<missing>:0
unknown function (ip: 0x7fad186e0682)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
showerror at ./task.jl:26
unknown function (ip: 0x7fad186e0346)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#476 at ./replutil.jl:213
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#with_output_color#512 at ./util.jl:400
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
with_output_color at ./util.jl:397
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#showerror#475 at ./replutil.jl:212
unknown function (ip: 0x7fad186df389)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#showerror at ./<missing>:0
unknown function (ip: 0x7fad186df0f2)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#showerror#478 at ./replutil.jl:222
unknown function (ip: 0x7fad186deda9)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1424 [inlined]
jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:51
showerror at ./replutil.jl:221
unknown function (ip: 0x7fad186deb7d)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
display_error at ./client.jl:137
unknown function (ip: 0x7fad186db3ed)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
display_error at ./client.jl:140
unknown function (ip: 0x7fad186dae66)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
do_call at /buildworker/worker/package_linux64/build/src/interpreter.c:75
eval at /buildworker/worker/package_linux64/build/src/interpreter.c:242
eval_body at /buildworker/worker/package_linux64/build/src/interpreter.c:539
jl_toplevel_eval_body at /buildworker/worker/package_linux64/build/src/interpreter.c:511
jl_toplevel_eval_flex at /buildworker/worker/package_linux64/build/src/toplevel.c:571
jl_toplevel_eval_in at /buildworker/worker/package_linux64/build/src/builtins.c:496
eval at ./boot.jl:235
unknown function (ip: 0x7fad48eadacf)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
_start at ./client.jl:417
unknown function (ip: 0x7fad48effb28)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
unknown function (ip: 0x401bc8)
unknown function (ip: 0x401612)
__libc_start_main at /build/glibc-bfm8X4/glibc-2.23/csu/../csu/libc-start.c:291
unknown function (ip: 0x4016bc)

However, if I only have @variable(model, x) in the model instead of @variable(model, x[i in I; i <=2]>=1), it will work well.

Thanks!

Best,
Can

2

getvalue in the case above returns a JuMPDict that has a reference to a JuMP model, so pmap will eventually try to serialize a JuMP Model, which isn’t supported. Copy the result into a simple data structure like a Dict or an Array if you’re going to send values across processes.

3

Hi Miles,

Thanks a lot for the answer. Could you be more specific about how to “copy the result into a simple data structure like a Dict or an Array” when I use pmap to solve the above problem? Where should I apply getvalue() to get the value of the optimal solution in each processor? Thanks!

Best regards,
Can

4

Provided I is a one-indexed unit-range, instead of

d[:x] = getvalue(getindex(m, :x))

go

d[:x] = getvalue(getindex(m, :x))[:]

This way d[:x] is of type Vector{Float64} instead of JuMPDict. Otherwise you could go

d[:x] = Dict()
xval = getvalue(getindex(m, :x)) # xval is type JuMPDict
for i in I
    d[:x][i] = xval[i]
end
# but now d[:x] is just a normal Julia Dict.
5

FWIW this won’t be an issue on JuMP 0.19 because we’ve replaced JuMPDict with the standard Dict.

6

Hi Oscar and Miles,

Thanks for your answer. I tried replacing

d[:x] = getvalue(getindex(m, :x))

with

d[:x] = getvalue(getindex(m, :x))[:]

but I still get the error:

getindex at ./associative.jl:363
_map at /home/canl1/.julia/v0.6/JuMP/src/JuMPContainer.jl:162
getvalue at /home/canl1/.julia/v0.6/JuMP/src/JuMPContainer.jl:114
psolve at /home/canl1/work/hierarchy/large/3/dump/smallmodel.jl:16
#106 at ./distributed/process_messages.jl:268 [inlined]
run_work_thunk at ./distributed/process_messages.jl:56
macro expansion at ./distributed/process_messages.jl:268 [inlined]
#105 at ./event.jl:73
Stacktrace:
 [1] #571 at ./asyncmap.jl:178 [inlined]
 [2] foreach(::Base.##571#573, ::Array{Any,1}) at ./abstractarray.jl:1733
 [3] maptwice(::Function, ::Channel{Any}, ::Array{Any,1}, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./asyncmap.jl:178
 [4] wrap_n_exec_twice(::Channel{Any}, ::Array{Any,1}, ::Base.Distributed.##204#207{WorkerPool}, ::Function, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./asyncmap.jl:154
 [5] #async_usemap#556(::Function, ::Void, ::Function, ::Base.Distributed.##188#190, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./asyncmap.jl:103
 [6] (::Base.#kw##async_usemap)(::Array{Any,1}, ::Base.#async_usemap, ::Function, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./<missing>:0
 [7] (::Base.#kw##asyncmap)(::Array{Any,1}, ::Base.#asyncmap, ::Function, ::Array{Any,1}) at ./<missing>:0
 [8] #pmap#203(::Bool, ::Int64, ::Void, ::Array{Any,1}, ::Void, ::Function, ::WorkerPool, ::Function, ::Array{Any,1}) at ./distributed/pmap.jl:126
 [9] pmap(::WorkerPool, ::Function, ::Array{Any,1}) at ./distributed/pmap.jl:101
 [10] #pmap#213(::Array{Any,1}, ::Function, ::Function, ::Array{Any,1}) at ./distributed/pmap.jl:156
 [11] pmap(::Function, ::Array{Any,1}) at ./distributed/pmap.jl:156
 [12] include_from_node1(::String) at ./loading.jl:576
 [13] include(::String) at ./sysimg.jl:14
 [14] process_options(::Base.JLOptions) at ./client.jl:305
 [15] _start() at ./client.jl:371fatal: error thrown and no exception handler available.
Base.AssertionError(msg="j isa Array{Variable}")
7

Sorry, I misread the code. I also wouldn’t expect passing an array of models pmap to work. Don’t send models across processes. Send the input data and generate the models locally on the process.