Problem loading saved neural network

LucasMSpereira · May 15, 2023, 2:40pm

When trying to load a saved Flux NN with
BSON.@load datasetPath * "trainedNetworks/convnextGen.bson" cpuGenerator
I’m getting a invalid struct allocation error.

Since doing this last time, the only thing I did was update julia to 1.9.0 when it launched. In the meantime, BSON.jl and Flux.jl versions didn’t change, and nothing was done to the BSON file. What is the problem here?

Stacktrace:

ERROR: invalid struct allocation
Stacktrace:
   [1] newstruct(::Type, ::Module, ::Vararg{Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:135
   [2] (::BSON.var"#47#48")(d::Dict{Symbol, Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:159
   [3] _raise_recursive(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)    
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:82
   [4] (::BSON.var"#49#50")(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:168
   [5] raise_recursive(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:92
   [6] (::BSON.var"#23#24"{IdDict{Any, Any}, Module})(x::Dict{Symbol, Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:98
   [7] applychildren!(f::BSON.var"#23#24"{IdDict{Any, Any}, Module}, x::Vector{Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\BSON.jl:26
   [8] raise_recursive
     @ C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:98 [inlined]
   [9] (::BSON.var"#45#46"{IdDict{Any, Any}, Module})(x::Vector{Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:153
  [10] iterate
     @ .\generator.jl:47 [inlined]
  [11] collect_to!(dest::Vector{Symbol}, itr::Base.Generator{Vector{Any}, BSON.var"#45#46"{IdDict{Any, Any}, Module}}, offs::Int64, st::Int64)
     @ Base .\array.jl:840
  [12] collect_to_with_first!(dest::Vector{Symbol}, v1::Symbol, itr::Base.Generator{Vector{Any}, BSON.var"#45#46"{IdDict{Any, Any}, Module}}, st::Int64)
     @ Base .\array.jl:818
  [13] _collect(c::Vector{Any}, itr::Base.Generator{Vector{Any}, BSON.var"#45#46"{IdDict{Any, Any}, Module}}, #unused#::Base.EltypeUnknown, isz::Base.HasShape{1})
     @ Base .\array.jl:812
  [14] collect_similar(cont::Vector{Any}, itr::Base.Generator{Vector{Any}, BSON.var"#45#46"{IdDict{Any, Any}, Module}})
     @ Base .\array.jl:711
  [15] map(f::Function, A::Vector{Any})
     @ Base .\abstractarray.jl:3261
  [16] newstruct_raw(cache::IdDict{Any, Any}, T::Type, d::Dict{Symbol, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:153
  [17] (::BSON.var"#49#50")(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:169
  [18] raise_recursive(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:92
  [19] (::BSON.var"#23#24"{IdDict{Any, Any}, Module})(x::Dict{Symbol, Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:98
  [20] applychildren!(f::BSON.var"#23#24"{IdDict{Any, Any}, Module}, x::Vector{Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\BSON.jl:26
  [21] raise_recursive
     @ C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:98 [inlined]
  [22] (::BSON.var"#45#46"{IdDict{Any, Any}, Module})(x::Vector{Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:153
  [23] iterate
     @ .\generator.jl:47 [inlined]
  [24] _collect(c::Vector{Any}, itr::Base.Generator{Vector{Any}, BSON.var"#45#46"{IdDict{Any, Any}, Module}}, #unused#::Base.EltypeUnknown, isz::Base.HasShape{1})
     @ Base .\array.jl:802
  [25] collect_similar(cont::Vector{Any}, itr::Base.Generator{Vector{Any}, BSON.var"#45#46"{IdDict{Any, Any}, Module}})
     @ Base .\array.jl:711
  [26] map(f::Function, A::Vector{Any})
     @ Base .\abstractarray.jl:3261
  [27] newstruct_raw(cache::IdDict{Any, Any}, T::Type, d::Dict{Symbol, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:153
  [28] (::BSON.var"#49#50")(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:169
  [29] raise_recursive(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:92
  [30] (::BSON.var"#45#46"{IdDict{Any, Any}, Module})(x::Dict{Symbol, Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:153
  [31] iterate
     @ .\generator.jl:47 [inlined]
  [32] collect_to!(dest::Vector{Any}, itr::Base.Generator{Vector{Any}, BSON.var"#45#46"{IdDict{Any, Any}, Module}}, offs::Int64, st::Int64)
     @ Base .\array.jl:840
  [33] collect_to!(dest::Vector{Module}, itr::Base.Generator{Vector{Any}, BSON.var"#45#46"{IdDict{Any, Any}, Module}}, offs::Int64, st::Int64)
     @ Base .\array.jl:848
  [34] collect_to_with_first!(dest::Vector{Module}, v1::Module, itr::Base.Generator{Vector{Any}, BSON.var"#45#46"{IdDict{Any, Any}, Module}}, st::Int64)
     @ Base .\array.jl:818
  [35] _collect(c::Vector{Any}, itr::Base.Generator{Vector{Any}, BSON.var"#45#46"{IdDict{Any, Any}, Module}}, #unused#::Base.EltypeUnknown, isz::Base.HasShape{1})
     @ Base .\array.jl:812
  [36] collect_similar(cont::Vector{Any}, itr::Base.Generator{Vector{Any}, BSON.var"#45#46"{IdDict{Any, Any}, Module}})
     @ Base .\array.jl:711
  [37] map(f::Function, A::Vector{Any})
     @ Base .\abstractarray.jl:3261
  [38] newstruct_raw(cache::IdDict{Any, Any}, T::Type, d::Dict{Symbol, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:153
  [39] (::BSON.var"#49#50")(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:169
  [40] raise_recursive(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:92
  [41] (::BSON.var"#23#24"{IdDict{Any, Any}, Module})(x::Dict{Symbol, Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:98
  [42] applychildren!(f::BSON.var"#23#24"{IdDict{Any, Any}, Module}, x::Vector{Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\BSON.jl:26
  [43] raise_recursive
     @ C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:98 [inlined]
  [44] (::BSON.var"#23#24"{IdDict{Any, Any}, Module})(x::Vector{Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:98
  [45] applychildren!(f::BSON.var"#23#24"{IdDict{Any, Any}, Module}, x::Vector{Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\BSON.jl:26
  [46] raise_recursive(v::Vector{Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:98
  [47] newstruct_raw(cache::IdDict{Any, Any}, #unused#::Type{Core.TypeName}, d::Dict{Symbol, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\anonymous.jl:146
  [48] (::BSON.var"#49#50")(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:169
  [49] raise_recursive(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:92
  [50] (::BSON.var"#18#21"{IdDict{Any, Any}, Module})(x::Dict{Symbol, Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:82
  [51] applychildren!(f::BSON.var"#18#21"{IdDict{Any, Any}, Module}, x::Dict{Symbol, Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\BSON.jl:19
  [52] _raise_recursive(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:82
  [53] raise_recursive(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:93
  [54] (::BSON.var"#23#24"{IdDict{Any, Any}, Module})(x::Dict{Symbol, Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:98
  [55] applychildren!(f::BSON.var"#23#24"{IdDict{Any, Any}, Module}, x::Vector{Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\BSON.jl:26
  [56] raise_recursive
     @ C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:98 [inlined]
  [57] (::BSON.var"#17#20"{IdDict{Any, Any}, Module})(x::Vector{Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:80
  [58] applychildren!(f::BSON.var"#17#20"{IdDict{Any, Any}, Module}, x::Dict{Symbol, Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\BSON.jl:19
  [59] _raise_recursive(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:80
--- the last 7 lines are repeated 6 more times ---
 [102] raise_recursive(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:93
 [103] (::BSON.var"#49#50")(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\extensions.jl:167
 [104] raise_recursive(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:92
 [105] (::BSON.var"#19#22"{IdDict{Any, Any}, Module})(x::Dict{Symbol, Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:86
 [106] applychildren!(f::BSON.var"#19#22"{IdDict{Any, Any}, Module}, x::Dict{Symbol, Any})
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\BSON.jl:19
 [107] _raise_recursive(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:86
 [108] raise_recursive(d::Dict{Symbol, Any}, cache::IdDict{Any, Any}, init::Module)
     @ BSON C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:93
 [109] raise_recursive
     @ C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:103 [inlined]
 [110] load (repeats 2 times)
     @ C:\Users\kaoid\.julia\packages\BSON\73cTU\src\read.jl:108 [inlined]
 [111] macro expansion
     @ C:\Users\kaoid\.julia\packages\BSON\73cTU\src\BSON.jl:50 [inlined]
 [112] loadTrainedGANs(model::Symbol)
     @ Main c:\Users\kaoid\My Drive\Estudo\Poli\Pesquisa\Programas\QuickTO\QuickTO\utilities\IO\readIO.jl:156

jling · May 15, 2023, 6:15pm

did you remember to move model back to CPU before saving it?

LucasMSpereira · May 15, 2023, 6:17pm

Yes

jling · May 15, 2023, 6:20pm

oh, from docs

Saving models this way could lead to compatibility issues across julia versions and across Flux versions if some of the Flux layers’ internals are changed. It is therefore not recommended for long term storage, use Flux.state instead.

LucasMSpereira · May 15, 2023, 6:22pm

Oof. After so much time reading Flux’s docs, I forgot that . I’ll look in to it. thanks

ToucheSir · May 16, 2023, 12:42am

In fairness, those docs are brand new!

@LucasMSpereira, as a stopgap you can also load your model on 1.8, save them in another format (JLD2, native Julia Serialization, etc) and load them back in on 1.9. Those two will have a better chance of working than BSON.jl because the latter is unmaintained and often broken by new Julia versions.

LucasMSpereira · May 16, 2023, 1:20am

That’s great to know. I was going to just go back to 1.8.5 for this project, but I would miss the (much) faster package load times of 1.9.0. I’ll try those other methods then. Thanks

HenriDeh · May 16, 2023, 3:31pm

Had the same issue, I can confirm that reverting to 1.8.5, as frustrating as it is, works.

ToucheSir · May 16, 2023, 3:42pm

Why not try one of the two approaches above? It’s the way things are moving anyhow (because so many of us have been bitten by BSON.jl issues), and it should only take a few seconds.

HenriDeh · May 16, 2023, 4:09pm

Will do, thanks for the suggestion btw.