How to design to keep types inferrable?

You could do something like this

import Base.push!

struct MyModel{T <: Number}
    mats::Vector{Matrix{T}}
    diag_mats::Vector{Diagonal{T}}
end

MyModel() = MyModel( Vector{Matrix{Float64}}(), Vector{Diagonal{Float64}}() )

push!(m:: MyModel,mat::Matrix{T})   where T <: Number = push!(m.mats,mat)
push!(m:: MyModel,mat::Diagonal{T}) where T <: Number = push!(m.diag_mats,mat)

model = MyModel() 
push!(model,Diagonal([1.0; 2.0])) 
push!(model,[1.0 1.0; 1.0 1.0])

solve(m::MyModel) = m.mats[1] * m.diag_mats[1]

But I don’t know if you could then write your solve function in a nice way. You could use meta-programming to deal with larger number of matrix types.