Error solve JuMP models in parallel


#1

Hi folks,
I’m new to Julia. I’m solving some JuMP models in parallel and I encountered the following error. The smallest example that I can come up with is as follows:

smallmodel.jl

using JuMP
using Ipopt
function generate_model()
	model = Model(solver=IpoptSolver())
	I = 1:3
	@variable(model, x[i in I; i <=2]>=1)
	@objective(model, Min, sum(x[i] for i in I if i <=2))
	return model
end


function psolve(m::JuMP.Model)
	temp = solve(m)
	d = Dict()
	d[:status] = temp
	d[:x] = getvalue(getindex(m, :x))
	return d
end

In the smallmodel.jl, I define variable x whose index has to be i in I; i <=2. And a psolve function.

Then I try to solve the model both in sequel and in parallel.

addprocs(2)

@everywhere include("smallmodel.jl")

model_array = []
for i = 1:3
	push!(model_array, generate_model())
end

for i = 1:3
	solve(model_array[i])
	println(getvalue(getindex(model_array[i], :x)))
end
results = pmap(psolve, model_array)
println(results[3][:x])

The solve in sequel works well. However, if I solve in parallel, I encountered the following error with the getindex function

ERROR: LoadError: On worker 2:

getindex at ./associative.jl:363
_map at /home/canl1/.julia/v0.6/JuMP/src/JuMPContainer.jl:162
getvalue at /home/canl1/.julia/v0.6/JuMP/src/JuMPContainer.jl:114
psolve at /home/canl1/work/hierarchy/large/3/smallmodel.jl:16
#106 at ./distributed/process_messages.jl:268 [inlined]
run_work_thunk at ./distributed/process_messages.jl:56
macro expansion at ./distributed/process_messages.jl:268 [inlined]
#105 at ./event.jl:73
Stacktrace:
 [1] #571 at ./asyncmap.jl:178 [inlined]
 [2] foreach(::Base.##571#573, ::Array{Any,1}) at ./abstractarray.jl:1733
 [3] maptwice(::Function, ::Channel{Any}, ::Array{Any,1}, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./asyncmap.jl:178
 [4] wrap_n_exec_twice(::Channel{Any}, ::Array{Any,1}, ::Base.Distributed.##204#207{WorkerPool}, ::Function, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./asyncmap.jl:154
 [5] #async_usemap#556(::Function, ::Void, ::Function, ::Base.Distributed.##188#190, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./asyncmap.jl:103
 [6] (::Base.#kw##async_usemap)(::Array{Any,1}, ::Base.#async_usemap, ::Function, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./<missing>:0
 [7] (::Base.#kw##asyncmap)(::Array{Any,1}, ::Base.#asyncmap, ::Function, ::Array{Any,1}) at ./<missing>:0
 [8] #pmap#203(::Bool, ::Int64, ::Void, ::Array{Any,1}, ::Void, ::Function, ::WorkerPool, ::Function, ::Array{Any,1}) at ./distributed/pmap.jl:126
 [9] pmap(::WorkerPool, ::Function, ::Array{Any,1}) at ./distributed/pmap.jl:101
 [10] #pmap#213(::Array{Any,1}, ::Function, ::Function, ::Array{Any,1}) at ./distributed/pmap.jl:156
 [11] pmap(::Function, ::Array{Any,1}) at ./distributed/pmap.jl:156
 [12] include_from_node1(::String) at ./loading.jl:576
 [13] include(::String) at ./sysimg.jl:14
 [14] process_options(::Base.JLOptions) at ./client.jl:305
 [15] _start() at ./client.jl:371fatal: error thrown and no exception handler available.
Base.AssertionError(msg="j isa Array{Variable}")
rec_backtrace at /buildworker/worker/package_linux64/build/src/stackwalk.c:84
record_backtrace at /buildworker/worker/package_linux64/build/src/task.c:245
jl_throw at /buildworker/worker/package_linux64/build/src/task.c:564
cont_str at /home/canl1/.julia/v0.6/JuMP/src/print.jl:417
unknown function (ip: 0x7fad186e5e9d)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1424 [inlined]
jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:51
show at /home/canl1/.julia/v0.6/JuMP/src/print.jl:397 [inlined]
showall at ./show.jl:1719
repr at ./strings/io.jl:146
unknown function (ip: 0x7fad186e15e2)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
showerror at ./replutil.jl:282
#476 at ./replutil.jl:213
unknown function (ip: 0x7fad186e13e2)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#with_output_color#512 at ./util.jl:400
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
with_output_color at ./util.jl:397
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#showerror#475 at ./replutil.jl:212
unknown function (ip: 0x7fad186e0919)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#showerror at ./<missing>:0
unknown function (ip: 0x7fad186e0682)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
showerror at ./task.jl:26
unknown function (ip: 0x7fad186e0346)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#476 at ./replutil.jl:213
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#with_output_color#512 at ./util.jl:400
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
with_output_color at ./util.jl:397
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#showerror#475 at ./replutil.jl:212
unknown function (ip: 0x7fad186df389)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#showerror at ./<missing>:0
unknown function (ip: 0x7fad186df0f2)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
#showerror#478 at ./replutil.jl:222
unknown function (ip: 0x7fad186deda9)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1424 [inlined]
jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:51
showerror at ./replutil.jl:221
unknown function (ip: 0x7fad186deb7d)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
display_error at ./client.jl:137
unknown function (ip: 0x7fad186db3ed)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
display_error at ./client.jl:140
unknown function (ip: 0x7fad186dae66)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
do_call at /buildworker/worker/package_linux64/build/src/interpreter.c:75
eval at /buildworker/worker/package_linux64/build/src/interpreter.c:242
eval_body at /buildworker/worker/package_linux64/build/src/interpreter.c:539
jl_toplevel_eval_body at /buildworker/worker/package_linux64/build/src/interpreter.c:511
jl_toplevel_eval_flex at /buildworker/worker/package_linux64/build/src/toplevel.c:571
jl_toplevel_eval_in at /buildworker/worker/package_linux64/build/src/builtins.c:496
eval at ./boot.jl:235
unknown function (ip: 0x7fad48eadacf)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
_start at ./client.jl:417
unknown function (ip: 0x7fad48effb28)
jl_call_fptr_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:339 [inlined]
jl_call_method_internal at /buildworker/worker/package_linux64/build/src/julia_internal.h:358 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:1926
unknown function (ip: 0x401bc8)
unknown function (ip: 0x401612)
__libc_start_main at /build/glibc-bfm8X4/glibc-2.23/csu/../csu/libc-start.c:291
unknown function (ip: 0x4016bc)

However, if I only have @variable(model, x) in the model instead of @variable(model, x[i in I; i <=2]>=1), it will work well.

Thanks!

Best,
Can


#2

getvalue in the case above returns a JuMPDict that has a reference to a JuMP model, so pmap will eventually try to serialize a JuMP Model, which isn’t supported. Copy the result into a simple data structure like a Dict or an Array if you’re going to send values across processes.


#3

Hi Miles,

Thanks a lot for the answer. Could you be more specific about how to “copy the result into a simple data structure like a Dict or an Array” when I use pmap to solve the above problem? Where should I apply getvalue() to get the value of the optimal solution in each processor? Thanks!

Best regards,
Can


#4

Provided I is a one-indexed unit-range, instead of

d[:x] = getvalue(getindex(m, :x))

go

d[:x] = getvalue(getindex(m, :x))[:]

This way d[:x] is of type Vector{Float64} instead of JuMPDict. Otherwise you could go

d[:x] = Dict()
xval = getvalue(getindex(m, :x)) # xval is type JuMPDict
for i in I
    d[:x][i] = xval[i]
end
# but now d[:x] is just a normal Julia Dict.

#5

FWIW this won’t be an issue on JuMP 0.19 because we’ve replaced JuMPDict with the standard Dict.


#6

Hi Oscar and Miles,

Thanks for your answer. I tried replacing

d[:x] = getvalue(getindex(m, :x))

with

d[:x] = getvalue(getindex(m, :x))[:]

but I still get the error:

getindex at ./associative.jl:363
_map at /home/canl1/.julia/v0.6/JuMP/src/JuMPContainer.jl:162
getvalue at /home/canl1/.julia/v0.6/JuMP/src/JuMPContainer.jl:114
psolve at /home/canl1/work/hierarchy/large/3/dump/smallmodel.jl:16
#106 at ./distributed/process_messages.jl:268 [inlined]
run_work_thunk at ./distributed/process_messages.jl:56
macro expansion at ./distributed/process_messages.jl:268 [inlined]
#105 at ./event.jl:73
Stacktrace:
 [1] #571 at ./asyncmap.jl:178 [inlined]
 [2] foreach(::Base.##571#573, ::Array{Any,1}) at ./abstractarray.jl:1733
 [3] maptwice(::Function, ::Channel{Any}, ::Array{Any,1}, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./asyncmap.jl:178
 [4] wrap_n_exec_twice(::Channel{Any}, ::Array{Any,1}, ::Base.Distributed.##204#207{WorkerPool}, ::Function, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./asyncmap.jl:154
 [5] #async_usemap#556(::Function, ::Void, ::Function, ::Base.Distributed.##188#190, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./asyncmap.jl:103
 [6] (::Base.#kw##async_usemap)(::Array{Any,1}, ::Base.#async_usemap, ::Function, ::Array{Any,1}, ::Vararg{Array{Any,1},N} where N) at ./<missing>:0
 [7] (::Base.#kw##asyncmap)(::Array{Any,1}, ::Base.#asyncmap, ::Function, ::Array{Any,1}) at ./<missing>:0
 [8] #pmap#203(::Bool, ::Int64, ::Void, ::Array{Any,1}, ::Void, ::Function, ::WorkerPool, ::Function, ::Array{Any,1}) at ./distributed/pmap.jl:126
 [9] pmap(::WorkerPool, ::Function, ::Array{Any,1}) at ./distributed/pmap.jl:101
 [10] #pmap#213(::Array{Any,1}, ::Function, ::Function, ::Array{Any,1}) at ./distributed/pmap.jl:156
 [11] pmap(::Function, ::Array{Any,1}) at ./distributed/pmap.jl:156
 [12] include_from_node1(::String) at ./loading.jl:576
 [13] include(::String) at ./sysimg.jl:14
 [14] process_options(::Base.JLOptions) at ./client.jl:305
 [15] _start() at ./client.jl:371fatal: error thrown and no exception handler available.
Base.AssertionError(msg="j isa Array{Variable}")

#7

Sorry, I misread the code. I also wouldn’t expect passing an array of models pmap to work. Don’t send models across processes. Send the input data and generate the models locally on the process.


Parallel Solves in Gurobi.jl