Issue with Float32 Precision in Turing Model Sampling

Specific Domains Probabilistic programming
,
1

I’m encountering an issue when trying to use Float32 precision in a Turing.jl model. Despite ensuring all my data, neural network outputs, and model parameters are in Float32, I’m getting a type mismatch error during sampling.

Is there a way to configure Turing or AdvancedHMC to properly handle Float32 types? Are there known limitations or workarounds for using Float32 precision in these packages?

Any guidance or suggestions would be greatly appreciated!

@model function inverse_model(data)

    global st
    # Prior distribution
    D_z ~ Normal(1.9340f-08 , 1f-7) 

    ct = (layer_1 = (C_eq = 40.6205f0, h = 0.0015f0, D_z = D_z), layer_2 = NamedTuple(), 
    layer_3 = NamedTuple())


    predicted = predict_neuralode(y_0, eltype(D_z).(nn_param.u), ct)
    avg_concentration = mean(predicted, dims=1)[:]


    data[:] ~ MvNormal(avg_concentration,1)


    return nothing
end

model = inverse_model(true_ode_data)
chain = sample(model, NUTS(), MCMCSerial(), 200, 1; progress=true)

Stracktrace:

ERROR: MethodError: no method matching AdvancedHMC.PhasePoint(::Vector{Float32}, ::Vector{Float32}, 
::AdvancedHMC.DualValue{Float32, Vector{Float32}}, ::AdvancedHMC.DualValue{Float64, Vector{Float64}})

Closest candidates are:
  AdvancedHMC.PhasePoint(::T, ::T, ::V, ::V) where {T, V}
   @ AdvancedHMC C:\Users\bozdoc\.julia\packages\AdvancedHMC\LJv94\src\hamiltonian.jl:53

Stacktrace:
  [1] phasepoint(h::AdvancedHMC.Hamiltonian{AdvancedHMC.DiagEuclideanMetric{Float64, Vector{Float64}}, AdvancedHMC.GaussianKinetic, Base.Fix1{typeof(LogDensityProblems.logdensity), LogDensityProblemsADForwardDiffExt.ForwardDiffLogDensity{LogDensityFunction{DynamicPPL.TypedVarInfo{NamedTuple{(:D_z,), Tuple{DynamicPPL.Metadata{Dict{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}, Int64}, Vector{Normal{Float32}}, Vector{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}}, Vector{Float32}, Vector{Set{DynamicPPL.Selector}}}}}, Float64}, DynamicPPL.Model{typeof(inverse_model), (:data,), (), (), Tuple{Matrix{Float32}}, Tuple{}, DynamicPPL.DefaultContext}, DynamicPPL.SamplingContext{DynamicPPL.Sampler{NUTS{Turing.Essential.ForwardDiffAD{0}, (), AdvancedHMC.DiagEuclideanMetric}}, DynamicPPL.DefaultContext, TaskLocalRNG}}, ForwardDiff.Chunk{1}, ForwardDiff.Tag{Turing.TuringTag, Float32}, ForwardDiff.GradientConfig{ForwardDiff.Tag{Turing.TuringTag, Float32}, Float32, 1, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Turing.TuringTag, Float32}, Float32, 1}}}}}, Turing.Inference.var"#∂logπ∂θ#36"{LogDensityProblemsADForwardDiffExt.ForwardDiffLogDensity{LogDensityFunction{DynamicPPL.TypedVarInfo{NamedTuple{(:D_z,), Tuple{DynamicPPL.Metadata{Dict{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}, Int64}, Vector{Normal{Float32}}, Vector{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}}, Vector{Float32}, Vector{Set{DynamicPPL.Selector}}}}}, Float64}, DynamicPPL.Model{typeof(inverse_model), (:data,), (), (), Tuple{Matrix{Float32}}, Tuple{}, DynamicPPL.DefaultContext}, DynamicPPL.SamplingContext{DynamicPPL.Sampler{NUTS{Turing.Essential.ForwardDiffAD{0}, (), AdvancedHMC.DiagEuclideanMetric}}, DynamicPPL.DefaultContext, TaskLocalRNG}}, ForwardDiff.Chunk{1}, ForwardDiff.Tag{Turing.TuringTag, Float32}, ForwardDiff.GradientConfig{ForwardDiff.Tag{Turing.TuringTag, Float32}, Float32, 1, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Turing.TuringTag, Float32}, Float32, 1}}}}}}, θ::Vector{Float32}, _r::Vector{Float64}; r::Vector{Float32}, ℓπ::AdvancedHMC.DualValue{Float32, Vector{Float32}}, ℓκ::AdvancedHMC.DualValue{Float64, Vector{Float64}})
    @ AdvancedHMC C:\Users\bozdoc\.julia\packages\AdvancedHMC\LJv94\src\hamiltonian.jl:92
  [2] phasepoint(h::AdvancedHMC.Hamiltonian{AdvancedHMC.DiagEuclideanMetric{Float64, Vector{Float64}}, AdvancedHMC.GaussianKinetic, Base.Fix1{typeof(LogDensityProblems.logdensity), LogDensityProblemsADForwardDiffExt.ForwardDiffLogDensity{LogDensityFunction{DynamicPPL.TypedVarInfo{NamedTuple{(:D_z,), Tuple{DynamicPPL.Metadata{Dict{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}, Int64}, Vector{Normal{Float32}}, Vector{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}}, Vector{Float32}, Vector{Set{DynamicPPL.Selector}}}}}, Float64}, DynamicPPL.Model{typeof(inverse_model), (:data,), (), (), Tuple{Matrix{Float32}}, Tuple{}, DynamicPPL.DefaultContext}, DynamicPPL.SamplingContext{DynamicPPL.Sampler{NUTS{Turing.Essential.ForwardDiffAD{0}, (), AdvancedHMC.DiagEuclideanMetric}}, DynamicPPL.DefaultContext, TaskLocalRNG}}, ForwardDiff.Chunk{1}, ForwardDiff.Tag{Turing.TuringTag, Float32}, ForwardDiff.GradientConfig{ForwardDiff.Tag{Turing.TuringTag, Float32}, Float32, 1, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Turing.TuringTag, Float32}, Float32, 1}}}}}, Turing.Inference.var"#∂logπ∂θ#36"{LogDensityProblemsADForwardDiffExt.ForwardDiffLogDensity{LogDensityFunction{DynamicPPL.TypedVarInfo{NamedTuple{(:D_z,), Tuple{DynamicPPL.Metadata{Dict{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}, Int64}, Vector{Normal{Float32}}, Vector{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}}, Vector{Float32}, Vector{Set{DynamicPPL.Selector}}}}}, Float64}, DynamicPPL.Model{typeof(inverse_model), (:data,), (), (), Tuple{Matrix{Float32}}, Tuple{}, DynamicPPL.DefaultContext}, DynamicPPL.SamplingContext{DynamicPPL.Sampler{NUTS{Turing.Essential.ForwardDiffAD{0}, (), AdvancedHMC.DiagEuclideanMetric}}, DynamicPPL.DefaultContext, TaskLocalRNG}}, ForwardDiff.Chunk{1}, ForwardDiff.Tag{Turing.TuringTag, Float32}, ForwardDiff.GradientConfig{ForwardDiff.Tag{Turing.TuringTag, Float32}, Float32, 1, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Turing.TuringTag, Float32}, Float32, 1}}}}}}, θ::Vector{Float32}, _r::Vector{Float64})
    @ AdvancedHMC C:\Users\bozdoc\.julia\packages\AdvancedHMC\LJv94\src\hamiltonian.jl:92
  [3] phasepoint(rng::TaskLocalRNG, θ::Vector{Float32}, h::AdvancedHMC.Hamiltonian{AdvancedHMC.DiagEuclideanMetric{Float64, Vector{Float64}}, AdvancedHMC.GaussianKinetic, Base.Fix1{typeof(LogDensityProblems.logdensity), LogDensityProblemsADForwardDiffExt.ForwardDiffLogDensity{LogDensityFunction{DynamicPPL.TypedVarInfo{NamedTuple{(:D_z,), Tuple{DynamicPPL.Metadata{Dict{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}, Int64}, Vector{Normal{Float32}}, Vector{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}}, Vector{Float32}, Vector{Set{DynamicPPL.Selector}}}}}, Float64}, DynamicPPL.Model{typeof(inverse_model), (:data,), (), (), Tuple{Matrix{Float32}}, Tuple{}, DynamicPPL.DefaultContext}, DynamicPPL.SamplingContext{DynamicPPL.Sampler{NUTS{Turing.Essential.ForwardDiffAD{0}, (), AdvancedHMC.DiagEuclideanMetric}}, DynamicPPL.DefaultContext, TaskLocalRNG}}, ForwardDiff.Chunk{1}, ForwardDiff.Tag{Turing.TuringTag, Float32}, ForwardDiff.GradientConfig{ForwardDiff.Tag{Turing.TuringTag, Float32}, 
Float32, 1, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Turing.TuringTag, Float32}, Float32, 1}}}}}, Turing.Inference.var"#∂logπ∂θ#36"{LogDensityProblemsADForwardDiffExt.ForwardDiffLogDensity{LogDensityFunction{DynamicPPL.TypedVarInfo{NamedTuple{(:D_z,), Tuple{DynamicPPL.Metadata{Dict{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}, Int64}, Vector{Normal{Float32}}, Vector{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}}, Vector{Float32}, Vector{Set{DynamicPPL.Selector}}}}}, Float64}, DynamicPPL.Model{typeof(inverse_model), (:data,), (), (), Tuple{Matrix{Float32}}, Tuple{}, DynamicPPL.DefaultContext}, DynamicPPL.SamplingContext{DynamicPPL.Sampler{NUTS{Turing.Essential.ForwardDiffAD{0}, (), AdvancedHMC.DiagEuclideanMetric}}, DynamicPPL.DefaultContext, TaskLocalRNG}}, ForwardDiff.Chunk{1}, ForwardDiff.Tag{Turing.TuringTag, Float32}, ForwardDiff.GradientConfig{ForwardDiff.Tag{Turing.TuringTag, Float32}, Float32, 1, Vector{ForwardDiff.Dual{ForwardDiff.Tag{Turing.TuringTag, Float32}, Float32, 
1}}}}}})
    @ AdvancedHMC C:\Users\bozdoc\.julia\packages\AdvancedHMC\LJv94\src\hamiltonian.jl:159
  [4] initialstep(rng::TaskLocalRNG, model::DynamicPPL.Model{typeof(inverse_model), (:data,), (), (), Tuple{Matrix{Float32}}, Tuple{}, DynamicPPL.DefaultContext}, spl::DynamicPPL.Sampler{NUTS{Turing.Essential.ForwardDiffAD{0}, (), AdvancedHMC.DiagEuclideanMetric}}, vi::DynamicPPL.TypedVarInfo{NamedTuple{(:D_z,), Tuple{DynamicPPL.Metadata{Dict{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}, Int64}, Vector{Normal{Float32}}, Vector{AbstractPPL.VarName{:D_z, Setfield.IdentityLens}}, Vector{Float32}, Vector{Set{DynamicPPL.Selector}}}}}, Float64}; init_params::Nothing, nadapts::Int64, kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Turing.Inference C:\Users\bozdoc\.julia\packages\Turing\UCuzt\src\mcmc\hmc.jl:164
  [5] step(rng::TaskLocalRNG, model::DynamicPPL.Model{typeof(inverse_model), (:data,), (), (), Tuple{Matrix{Float32}}, Tuple{}, DynamicPPL.DefaultContext}, spl::DynamicPPL.Sampler{NUTS{Turing.Essential.ForwardDiffAD{0}, (), AdvancedHMC.DiagEuclideanMetric}}; resume_from::Nothing, init_params::Nothing, kwargs::Base.Pairs{Symbol, Int64, Tuple{Symbol}, NamedTuple{(:nadapts,), Tuple{Int64}}})
    @ DynamicPPL C:\Users\bozdoc\.julia\packages\DynamicPPL\oX6N7\src\sampler.jl:111
  [6] step
    @ C:\Users\bozdoc\.julia\packages\DynamicPPL\oX6N7\src\sampler.jl:84 [inlined]
  [7] macro expansion
    @ C:\Users\bozdoc\.julia\packages\AbstractMCMC\fWWW0\src\sample.jl:125 [inlined]
  [8] macro expansion
    @ C:\Users\bozdoc\.julia\packages\ProgressLogging\6KXlp\src\ProgressLogging.jl:328 [inlined]    
  [9] macro expansion
    @ C:\Users\bozdoc\.julia\packages\AbstractMCMC\fWWW0\src\logging.jl:9 [inlined]
 [10] mcmcsample(rng::TaskLocalRNG, model::DynamicPPL.Model{typeof(inverse_model), (:data,), (), (), Tuple{Matrix{Float32}}, Tuple{}, DynamicPPL.DefaultContext}, sampler::DynamicPPL.Sampler{NUTS{Turing.Essential.ForwardDiffAD{0}, (), AdvancedHMC.DiagEuclideanMetric}}, N::Int64; progress::Bool, progressname::String, callback::Nothing, discard_initial::Int64, thinning::Int64, chain_type::Type, kwargs::Base.Pairs{Symbol, Union{Nothing, Int64}, Tuple{Symbol, Symbol}, NamedTuple{(:nadapts, :init_params), Tuple{Int64, Nothing}}})
    @ AbstractMCMC C:\Users\bozdoc\.julia\packages\AbstractMCMC\fWWW0\src\sample.jl:116
 [11] mcmcsample
    @ C:\Users\bozdoc\.julia\packages\AbstractMCMC\fWWW0\src\sample.jl:95 [inlined]
 [12] #sample#34
    @ C:\Users\bozdoc\.julia\packages\Turing\UCuzt\src\mcmc\hmc.jl:121 [inlined]
 [13] sample
    @ C:\Users\bozdoc\.julia\packages\Turing\UCuzt\src\mcmc\hmc.jl:91 [inlined]
 [14] (::AbstractMCMC.var"#sample_chain#78"{String, Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:chain_type, :progress), Tuple{UnionAll, Bool}}}, TaskLocalRNG, DynamicPPL.Model{typeof(inverse_model), (:data,), (), (), Tuple{Matrix{Float32}}, Tuple{}, DynamicPPL.DefaultContext}, DynamicPPL.Sampler{NUTS{Turing.Essential.ForwardDiffAD{0}, (), AdvancedHMC.DiagEuclideanMetric}}, Int64, Int64})(i::Int64, seed::UInt64, init_params::Nothing)
    @ AbstractMCMC C:\Users\bozdoc\.julia\packages\AbstractMCMC\fWWW0\src\sample.jl:511
 [15] sample_chain
    @ C:\Users\bozdoc\.julia\packages\AbstractMCMC\fWWW0\src\sample.jl:508 [inlined]
 [16] #4
    @ .\generator.jl:36 [inlined]
 [17] iterate
    @ .\generator.jl:47 [inlined]
 [18] collect(itr::Base.Generator{Base.Iterators.Zip{Tuple{UnitRange{Int64}, Vector{UInt64}}}, Base.var"#4#5"{AbstractMCMC.var"#sample_chain#78"{String, Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, 
NamedTuple{(:chain_type, :progress), Tuple{UnionAll, Bool}}}, TaskLocalRNG, DynamicPPL.Model{typeof(inverse_model), (:data,), (), (), Tuple{Matrix{Float32}}, Tuple{}, DynamicPPL.DefaultContext}, DynamicPPL.Sampler{NUTS{Turing.Essential.ForwardDiffAD{0}, (), AdvancedHMC.DiagEuclideanMetric}}, Int64, 
Int64}}})
    @ Base .\array.jl:782
 [19] map
    @ .\abstractarray.jl:3385 [inlined]
 [20] mcmcsample(rng::TaskLocalRNG, model::DynamicPPL.Model{typeof(inverse_model), (:data,), (), (), Tuple{Matrix{Float32}}, Tuple{}, DynamicPPL.DefaultContext}, sampler::DynamicPPL.Sampler{NUTS{Turing.Essential.ForwardDiffAD{0}, (), AdvancedHMC.DiagEuclideanMetric}}, ::MCMCSerial, N::Int64, nchains::Int64; progressname::String, init_params::Nothing, kwargs::Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:chain_type, :progress), Tuple{UnionAll, Bool}}})
    @ AbstractMCMC C:\Users\bozdoc\.julia\packages\AbstractMCMC\fWWW0\src\sample.jl:523
 [21] sample(rng::TaskLocalRNG, model::DynamicPPL.Model{typeof(inverse_model), (:data,), (), (), Tuple{Matrix{Float32}}, Tuple{}, DynamicPPL.DefaultContext}, sampler::DynamicPPL.Sampler{NUTS{Turing.Essential.ForwardDiffAD{0}, (), AdvancedHMC.DiagEuclideanMetric}}, ensemble::MCMCSerial, N::Int64, n_chains::Int64; chain_type::Type, progress::Bool, kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Turing.Inference C:\Users\bozdoc\.julia\packages\Turing\UCuzt\src\mcmc\Inference.jl:265       
 [22] sample
    @ C:\Users\bozdoc\.julia\packages\Turing\UCuzt\src\mcmc\Inference.jl:254 [inlined]
 [23] #sample#6
    @ C:\Users\bozdoc\.julia\packages\Turing\UCuzt\src\mcmc\Inference.jl:250 [inlined]
 [24] sample
    @ C:\Users\bozdoc\.julia\packages\Turing\UCuzt\src\mcmc\Inference.jl:241 [inlined]
 [25] #sample#5
    @ C:\Users\bozdoc\.julia\packages\Turing\UCuzt\src\mcmc\Inference.jl:237 [inlined]
 [26] top-level scope
    @ c:\Users\bozdoc\Desktop\CanRepo\Phase-1\neural_ODEq.jl:113
2

This is unfortunately not so easy to fix at the moment :confused:

There is nothing technical prohibiting us from running the model with Float32, but atm there are too many pieces of code with Float64 hardcoded in the Turing.jl. At some point we’ll go address this, but we haven’t gotten around to it yet :confused:

1 Like