@odow Thank you for pointing out this issue! I wasn’t aware of that detail about summarize size.
I understand that problems of this size are difficult to solve. However, for some large models it’s still possible to achieve a limited optimality gap.
@WalterMadelim I also recognize all of your points.
However, I still have the feeling that (some characteristic of) JuMP contributes significantly to the high memory usage. The example below builds the model in two different ways: one using JuMP and one using the C API. The final memory usage of both models is similar. However, the maximum memory usage of the Gurobi environment is twice as high when using JuMP. I also observe this when monitoring the Julia process trough the task manager.
Do you have any idea what might be causing the higher memory usage? Is there a way to reduce the maximum memory usage without building the entire model using the C API?
using StatsBase
using JuMP
using Gurobi
using MathOptInterface
function obtain_new_columns(num_nodes, num_routes)
new_columns= Vector{Vector{Int64}}()
for i=1:num_routes
route= sample(1:num_nodes, 20; replace= false)
push!(new_columns, route)
end
return new_columns
end
function model_jump(routes, num_nodes)
model = direct_model(Gurobi.Optimizer())
set_string_names_on_creation(model, false)
x= Vector{VariableRef}()
@constraint(model, constr[i=1:num_nodes], 0 <= 1)
for route in routes
push!(x, @variable(model, binary=true))
set_objective_coefficient(model, x[end], 1)
for i in route
set_normalized_coefficient(constr[i], x[end], 1)
end
end
@constraint(model, sum(x) <= 0)
optimize!(model)
return MOI.get(model, Gurobi.ModelAttribute("MemUsed")), MOI.get(model, Gurobi.ModelAttribute("MaxMemUsed"))
end
function model_api(routes, num_nodes)
env_p= Ref{Ptr{Cvoid}}()
error = Gurobi.GRBemptyenv(env_p)
env= env_p[]
error= GRBstartenv(env)
model_p = Ref{Ptr{Cvoid}}()
error= GRBnewmodel(env, model_p, "test", 0, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL)
model = model_p[]
for i=1:num_nodes
rhs=1.0
error= GRBaddconstr(model, 0, Cint[], Cdouble[], GRB_LESS_EQUAL, rhs, C_NULL)
end
#=
for (i,route) in enumerate(routes)
ind= Cint[(route .-1)...]
val= Cdouble[1 for i=1:length(route)]
nzc = length(ind)
error = GRBaddvar(model, nzc, ind, val, 1.0, 0.0, 1.0, GRB_BINARY, C_NULL)
end
=#
for (i,route) in enumerate(routes)
GRBaddvar(model, 0, Cint[], Cdouble[], 0.0, 0, 1, GRB_BINARY, C_NULL)
error = GRBsetdblattrelement(model, "Obj", i-1, 1.0)
for j in route
ind= Cint[j-1]
var= Cint[i-1]
val= Cdouble[1]
nzc= length(ind)
error= GRBchgcoeffs(model, nzc, ind, var, val)
end
end
error= GRBaddconstr(model, length(routes), Cint[i for i=0:length(routes)], Cdouble[1.0 for i=1:length(routes)], GRB_LESS_EQUAL, 1.0, C_NULL)
error = GRBoptimize(model)
mem_used= Ref{Cdouble}()
error= GRBgetdblattr(model, "MemUsed", mem_used)
mem_used = mem_used[]
max_mem_used= Ref{Cdouble}()
error= GRBgetdblattr(model, "MaxMemUsed", max_mem_used)
max_mem_used = max_mem_used[]
return mem_used, max_mem_used
end
Random.seed!(1234)
num_nodes= 100
# compile
routes= obtain_new_columns(num_nodes, 100)
@time mem_used_jump, max_mem_used_jump= model_jump(routes, num_nodes)
@time mem_used_api, max_mem_used_api= model_api(routes, num_nodes)
# test
routes= obtain_new_columns(num_nodes, 5000000)
@time mem_used_jump, max_mem_used_jump= model_jump(routes, num_nodes)
println("Jump: Memory used: ", mem_used_jump, " GB, Max memory used: ", max_mem_used_jump, " GB")
@time mem_used_api, max_mem_used_api= model_api(routes, num_nodes)
println("API: Memory used: ", mem_used_api, " GB, Max memory used: ", max_mem_used_api, " GB")
which gives the output
66.555772 seconds (540.00 M allocations: 11.572 GiB, 4.37% gc time)
Jump: Memory used: 5.199141979 GB, Max memory used: 9.610150477 GB
55.942253 seconds (310.00 M allocations: 18.533 GiB, 13.41% gc time)
API: Memory used: 4.963454876 GB, Max memory used: 5.223455684 GB