I am trying to solve relatively large systems of coupled ODEs, but I cannot seem to solve an EnsembleProblem(..) when the ODEProblem is generated using the @reaction_network macro of Catalyst.jl (or, more specifically, when the ODEProblem is generated using Reaction(..)s).
I have found a MWE that generates the following error:
UndefVarError: `#630#generated_observed#555` not defined
Full error message
ERROR: On worker 2:
UndefVarError: `#630#generated_observed#555` not defined
Stacktrace:
[1] deserialize_datatype
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:1385
[2] handle_deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:869
[3] deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:816
[4] deserialize_datatype
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:1409
[5] handle_deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:869
[6] deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:816
[7] deserialize_datatype
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:1409
[8] handle_deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:869
[9] deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:816
[10] deserialize_datatype
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:1409
[11] handle_deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:869
[12] deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:816
[13] deserialize_datatype
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:1409
[14] handle_deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:869
[15] deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:816
[16] handle_deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:876
[17] deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:816
[18] #5
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:975
[19] ntupleany
@ ./ntuple.jl:43
[20] deserialize_tuple
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:975
[21] handle_deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:859
[22] deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:816
[23] #5
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:975
[24] ntupleany
@ ./ntuple.jl:43
[25] deserialize_tuple
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:975
[26] handle_deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:859
[27] deserialize
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Serialization/src/Serialization.jl:816 [inlined]
[28] deserialize_msg
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Distributed/src/messages.jl:87
[29] #invokelatest#2
@ ./essentials.jl:819 [inlined]
[30] invokelatest
@ ./essentials.jl:816 [inlined]
[31] message_handler_loop
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Distributed/src/process_messages.jl:176
[32] process_tcp_streams
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Distributed/src/process_messages.jl:133
[33] #103
@ ./task.jl:514
Stacktrace:
[1] (::Base.var"#988#990")(x::Task)
@ Base ./asyncmap.jl:177
[2] foreach(f::Base.var"#988#990", itr::Vector{Any})
@ Base ./abstractarray.jl:3075
[3] maptwice(wrapped_f::Function, chnl::Channel{Any}, worker_tasks::Vector{Any}, c::UnitRange{Int64})
@ Base ./asyncmap.jl:177
[4] wrap_n_exec_twice
@ ./asyncmap.jl:153 [inlined]
[5] #async_usemap#973
@ ./asyncmap.jl:103 [inlined]
[6] async_usemap
@ ./asyncmap.jl:84 [inlined]
[7] #asyncmap#972
@ ./asyncmap.jl:81 [inlined]
[8] asyncmap
@ ./asyncmap.jl:80 [inlined]
[9] pmap(f::Function, p::CachingPool, c::UnitRange{Int64}; distributed::Bool, batch_size::Int64, on_error::Nothing, retry_delays::Vector{Any}, retry_check::Nothing)
@ Distributed ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Distributed/src/pmap.jl:126
[10] pmap
@ ~/.julia/juliaup/julia-1.9.3+0.x64.linux.gnu/share/julia/stdlib/v1.9/Distributed/src/pmap.jl:99 [inlined]
[11] solve_batch(prob::EnsembleProblem{ODEProblem{Vector{Float64}, Tuple{Float64, Float64}, true, Vector{Float64}, ODEFunction{true, SciMLBase.AutoSpecialize, ModelingToolkit.var"#k#545"{RuntimeGeneratedFunctions.RuntimeGeneratedFunction{(:ˍ₋arg1, :ˍ₋arg2, :t), ModelingToolkit.var"#_RGF_ModTag", ModelingToolkit.var"#_RGF_ModTag", (0x0d09c12c, 0x8e4e7937, 0x340bf229, 0xf16d82a9, 0x5a0eeb93), Nothing}, RuntimeGeneratedFunctions.RuntimeGeneratedFunction{(:ˍ₋out, :ˍ₋arg1, :ˍ₋arg2, :t), ModelingToolkit.var"#_RGF_ModTag", ModelingToolkit.var"#_RGF_ModTag", (0xb21fc0aa, 0x52e6e717, 0x30dbc817, 0xc96db097, 0x5ff42495), Nothing}}, LinearAlgebra.UniformScaling{Bool}, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Vector{Symbol}, Symbol, Vector{Symbol}, ModelingToolkit.var"#630#generated_observed#555"{Bool, ODESystem, Dict{Any, Any}, Vector{SymbolicUtils.BasicSymbolic{Real}}}, Nothing, ODESystem}, Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}, SciMLBase.StandardODEProblem}, typeof(prob_func), typeof(SciMLBase.DEFAULT_OUTPUT_FUNC), typeof(SciMLBase.DEFAULT_REDUCTION), Nothing}, alg::Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, ensemblealg::EnsembleDistributed, II::UnitRange{Int64}, pmap_batch_size::Int64; kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ SciMLBase ~/.julia/packages/SciMLBase/kTUaf/src/ensemble/basic_ensemble_solve.jl:136
[12] solve_batch
@ ~/.julia/packages/SciMLBase/kTUaf/src/ensemble/basic_ensemble_solve.jl:124 [inlined]
[13] macro expansion
@ ./timing.jl:393 [inlined]
[14] __solve(prob::EnsembleProblem{ODEProblem{Vector{Float64}, Tuple{Float64, Float64}, true, Vector{Float64}, ODEFunction{true, SciMLBase.AutoSpecialize, ModelingToolkit.var"#k#545"{RuntimeGeneratedFunctions.RuntimeGeneratedFunction{(:ˍ₋arg1, :ˍ₋arg2, :t), ModelingToolkit.var"#_RGF_ModTag", ModelingToolkit.var"#_RGF_ModTag", (0x0d09c12c, 0x8e4e7937, 0x340bf229, 0xf16d82a9, 0x5a0eeb93), Nothing}, RuntimeGeneratedFunctions.RuntimeGeneratedFunction{(:ˍ₋out, :ˍ₋arg1, :ˍ₋arg2, :t), ModelingToolkit.var"#_RGF_ModTag", ModelingToolkit.var"#_RGF_ModTag", (0xb21fc0aa, 0x52e6e717, 0x30dbc817, 0xc96db097, 0x5ff42495), Nothing}}, LinearAlgebra.UniformScaling{Bool}, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Vector{Symbol}, Symbol, Vector{Symbol}, ModelingToolkit.var"#630#generated_observed#555"{Bool, ODESystem, Dict{Any, Any}, Vector{SymbolicUtils.BasicSymbolic{Real}}}, Nothing, ODESystem}, Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}, SciMLBase.StandardODEProblem}, typeof(prob_func), typeof(SciMLBase.DEFAULT_OUTPUT_FUNC), typeof(SciMLBase.DEFAULT_REDUCTION), Nothing}, alg::Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, ensemblealg::EnsembleDistributed; trajectories::Int64, batch_size::Int64, pmap_batch_size::Int64, kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ SciMLBase ~/.julia/packages/SciMLBase/kTUaf/src/ensemble/basic_ensemble_solve.jl:64
[15] #solve#32
@ ~/.julia/packages/DiffEqBase/9rTlH/src/solve.jl:996 [inlined]
[16] top-level scope
@ REPL[36]:1
A MWE that generates this error is the following:
julia> using Catalyst, Distributed, DifferentialEquations
julia> addprocs(3, exeflags="--project=$(Base.active_project())")
julia> @everywhere using DifferentialEquations, Catalyst, ModelingToolkit
julia> rn = @reaction_network begin
b, 0 --> X
d, X --> 0
end
julia> u0 = [:X => 1.0]
julia> tspan = (0.0, 10.0)
julia> params = [:b => 1.0, :d => 0.2]
julia> prob = ODEProblem(rn, u0, tspan, params)
julia> @everywhere function prob_func(prob, i, repeat)
remake(prob, u0 = rand() * prob.u0)
end
julia> eprob = EnsembleProblem(prob, prob_func=prob_func)
julia> sol = solve(eprob, Tsit5(), EnsembleDistributed(), trajectories = 8)
I am breaking my head around this problem, as 1. I cannot find this particular error message anywhere, and 2. it does work when the ODEProblem is defined using a function directly, e.g. as in this example.
I am probably doing something dumb, but I cannot figure it out. Any help is appreciated.