# NeuralODE - Array has no field layer

Hi,

I’m new here and have a question about extrapolating data using neural ordinary differential equations. I am in the process of training a mathematical function in steps of four. Training the neural network seems to work. The parameters of the network are adjusted. However, I am not able to plot a prediction with the trained network.

Here is the code:

``````using Random, Dates, Optimization, ComponentArrays, Lux, OptimizationOptimisers, DiffEqFlux,
OrdinaryDiffEq, CSV, DataFrames, Dates, Statistics, Plots, DataDeps
using Flux

#----------------------------------Data Extraction--------------------------------------------------
time = 0:1:15
t = collect(range(extrema(time)..., length=length(time)))
tspan = (Float32.(t[1]), Float32.(t[end]))
#tspan = (Float32.(t_scale[1]), Float32.(t_scale[end]));
data = sin.(t).*exp.(-0.05*t)
plot(t, data)

u0 = [Float32(data[1])];

#-------------------------------Training NN------------------------------------------
function neural_ode(t, data_dim)
f = Lux.Chain(Lux.Dense(data_dim => 64, swish), Lux.Dense(64 => 32, swish), Lux.Dense(32 => data_dim))

node = NeuralODE(f, extrema(t), Tsit5(); saveat = t,
abstol = 1e-9, reltol = 1e-9)

rng = Random.default_rng()
p, state = Lux.setup(rng, f)

return node, ComponentArray(p), state
end

function train_one_round(node, p, state, y, opt, maxiters, rng, y0 = y[:, 1]; kwargs...)
predict(p) = Array(node(y0, p, state)[1])
loss(p) = sum(abs2, predict(p) .- y)

optf = OptimizationFunction((p, _) -> loss(p), adtype)
optprob = OptimizationProblem(optf, p)
res = solve(optprob, opt; maxiters = maxiters, kwargs...)
res.minimizer, state
end

function train(t, y, obs_grid, maxiters, lr, rng, p = nothing, state = nothing; kwargs...)
log_results(ps, losses) = (p, loss) -> begin
push!(ps, copy(p))
push!(losses, loss)
false
end

ps, losses = ComponentArray[], Float32[]
for k in obs_grid
node, p_new, state_new = neural_ode(t, size(y, 1))
p === nothing && (p = p_new)
state === nothing && (state = state_new)

p, state = train_one_round(node, p, state, y, ADAM(lr), maxiters, rng;
callback = log_results(ps, losses), kwargs...)
end
ps, state, losses
end

rng = MersenneTwister(123)
obs_grid = 4:4:length(time) # we train on an increasing amount of the first k obs
maxiters = 150
lr = 5e-3
ps, state, losses = train(time, data, obs_grid, maxiters, lr, rng; progress = true)

#-------------------Prediction Part--------------------------------
t_grid = Float32.(collect(range(minimum(time), maximum(time); length = 500)))
predict(y0, t, p, state) = begin
node, _, _ = neural_ode(t, length(y0))
Array(node(y0, p, state)[1])
end

y_pred = predict(u0, t_grid, ps, state)

plot(time, data, label="Originaldaten", xlabel="Zeit", ylabel="Daten", linewidth=2)
plot!(time, y_pred, label="Vorhersagen", linestyle=:dash, linewidth=2)
legend()
``````

I’m getting an error at this point:

``````predict(y0, t, p, state) = begin
node, _, _ = neural_ode(t, length(y0))
Array(node(y0, p, state)[1])
end

y_pred = predict(Float64(data[1]), t_grid, ps, state)
``````

ERROR: type Array has no field layer_1

Does anyone know how i can solve this problem?

It means the parameters you’re passing to the neural network are of type `Array` and not of type `ComponentArray`. Track down where the conversion was made or missing.

Thank you for your answer! I forgot to use the last parameter set of the training for the prediction. Here ist the solution:

``````predict(y0, t, p, state) = begin
node, _, _ = neural_ode(t, length(y0))
Array(node(y0, p, state)[1])
end

y_pred = predict(Float32(data[1]), t_grid, ps[end], state)

``````

Now I’m getting the next error at the same point:
ERROR: DimensionMismatch: second dimension of A, 16, does not match length
of x, 1

Apparently the prediction has a problem with the dimension of the vector “time” but I don’t know why.

Is it possible that the NODE is not able to learn a simple mathematical function?

I tried one code which uses 4 different time series of a weather forecast which means that the network hast 4 inputs of time and 4 outputs (Temperature, Humidity,Velocity, Pressure) of the meteorological aspects. This code works verry well.

``````using Random, Dates, Optimization, ComponentArrays, Lux, OptimizationOptimisers, DiffEqFlux,
OrdinaryDiffEq, CSV, DataFrames, Dates, Statistics, Plots, DataDeps

#----------------------------------Datenextraktion--------------------------------------------------
data_local_path = "./delhi")
end

return vcat(train_df, test_df)
end

#----------------------------------Daten ordnen--------------------------------------------------------------
FEATURES = [:meantemp, :humidity, :wind_speed, :meanpressure]
UNITS = ["Celsius", "g/m³ of water", "km/h", "hPa"]
FEATURE_NAMES = ["Mean temperature", "Humidity", "Wind speed", "Mean pressure"]

function plot_data(df)
plots = map(enumerate(zip(FEATURES, FEATURE_NAMES, UNITS))) do (i, (f, n, u))
plot(df[:, :date], df[:, f]; title = n, label = nothing,
ylabel = u, size = (800, 600), color = i)
end

n = length(plots)
plot(plots...; layout = (Int(n / 2), Int(n / 2)))
end

plot_data(df)

#------------------------------Daten standardisieren----------------------------------------
function standardize(x)
μ = mean(x; dims = 2)
σ = std(x; dims = 2)
z = (x .- μ) ./ σ
return z, μ, σ
end

function featurize(raw_df, num_train = 20)
raw_df.year = Float64.(year.(raw_df.date))
raw_df.month = Float64.(month.(raw_df.date))
df = combine(groupby(raw_df, [:year, :month]),
:date => (d -> mean(year.(d)) .+ mean(month.(d)) ./ 12),
:meantemp => mean,
:humidity => mean,
:wind_speed => mean,
:meanpressure => mean;
renamecols = false)
t_and_y(df) = df.date', Matrix(select(df, FEATURES))'
t_train, y_train = t_and_y(df[1:num_train, :])
t_test, y_test = t_and_y(df[(num_train + 1):end, :])
t_train, t_mean, t_scale = standardize(t_train)
y_train, y_mean, y_scale = standardize(y_train)
t_test = (t_test .- t_mean) ./ t_scale
y_test = (y_test .- y_mean) ./ y_scale

return (vec(t_train), y_train,
vec(t_test), y_test,
(t_mean, t_scale),
(y_mean, y_scale))
end

function plot_features(t_train, y_train, t_test, y_test)
plt_split = plot(reshape(t_train, :), y_train';
linewidth = 3, colors = 1:4,
xlabel = "Normalized time",
ylabel = "Normalized values",
label = nothing,
title = "Features")
plot!(plt_split, reshape(t_test, :), y_test';
linewidth = 3, linestyle = :dash,
color = [1 2 3 4], label = nothing)
plot!(plt_split, [0], [0]; linewidth = 0,
label = "Train", color = 1)
plot!(plt_split, [0], [0]; linewidth = 0,
linestyle = :dash, label = "Test",
color = 1,
ylims = (-5, 5))
end

t_train, y_train, t_test, y_test, (t_mean, t_scale), (y_mean, y_scale) = featurize(df)
plot_features(t_train, y_train, t_test, y_test)

#-------------------------------Training des NN------------------------------------------
function neural_ode(t, data_dim)
f = Lux.Chain(Lux.Dense(data_dim => 64, swish), Lux.Dense(64 => 32, swish), Lux.Dense(32 => data_dim))

node = NeuralODE(f, extrema(t), Tsit5(); saveat = t,
abstol = 1e-9, reltol = 1e-9)

rng = Random.default_rng()
p, state = Lux.setup(rng, f)

return node, ComponentArray(p), state
end

function train_one_round(node, p, state, y, opt, maxiters, rng, y0 = y[:, 1]; kwargs...)
predict(p) = Array(node(y0, p, state)[1])
loss(p) = sum(abs2, predict(p) .- y)

optf = OptimizationFunction((p, _) -> loss(p), adtype)
optprob = OptimizationProblem(optf, p)
res = solve(optprob, opt; maxiters = maxiters, kwargs...)
res.minimizer, state
end

function train(t, y, obs_grid, maxiters, lr, rng, p = nothing, state = nothing; kwargs...)
log_results(ps, losses) = (p, loss) -> begin
push!(ps, copy(p))
push!(losses, loss)
false
end

ps, losses = ComponentArray[], Float32[]
for k in obs_grid
node, p_new, state_new = neural_ode(t, size(y, 1))
p === nothing && (p = p_new)                #Löscht Eintrag von p, nimmt jedoch p für train_one_round
state === nothing && (state = state_new)

p, state = train_one_round(node, p, state, y, ADAM(lr), maxiters, rng;
callback = log_results(ps, losses), kwargs...)
end
ps, state, losses
end

rng = MersenneTwister(123)
obs_grid = 4:4:length(t_train) # we train on an increasing amount of the first k obs
maxiters = 150
lr = 5e-3
ps, state, losses = train(t_train, y_train, obs_grid, maxiters, lr, rng; progress = true)

#------------------------------Prediction Part-------------------------------------------
predict(y0, t, p, state) = begin
node, _, _ = neural_ode(t, length(y0))
Array(node(y0, p, state)[1])
end

node, _, _ = neural_ode(t, length(y_train[:, 1]))
y_pred = Array(node(y_train[:, 1], ps[end], state)[1])

t_grid = collect(range(minimum(t_train), maximum(t_test); length = 500))
y_pred = predict(y_train[:, 1], t_grid, ps[end], state)

plot(t_grid, y_pred[1,:], label = "Prediction")
plot!(t_train, y_train[1,:], label = "Data")
``````

Then I reduced the Inputs and Outputs to 1 (Only Temperature) and trained again. Now it does’nt work anymore.

``````using Random, Dates, Optimization, ComponentArrays, Lux, OptimizationOptimisers, DiffEqFlux,
OrdinaryDiffEq, CSV, DataFrames, Dates, Statistics, Plots, DataDeps

#----------------------------------Datenextraktion--------------------------------------------------
data_local_path = "./delhi")
end

return vcat(train_df, test_df)
end

#----------------------------------Daten ordnen--------------------------------------------------------------
FEATURES = [:meantemp, :humidity, :wind_speed, :meanpressure]
UNITS = ["Celsius", "g/m³ of water", "km/h", "hPa"]
FEATURE_NAMES = ["Mean temperature", "Humidity", "Wind speed", "Mean pressure"]

function plot_data(df)
plots = map(enumerate(zip(FEATURES, FEATURE_NAMES, UNITS))) do (i, (f, n, u))
plot(df[:, :date], df[:, f]; title = n, label = nothing,
ylabel = u, size = (800, 600), color = i)
end

n = length(plots)
plot(plots...; layout = (Int(n / 2), Int(n / 2)))
end

plot_data(df)

#------------------------------Daten standardisieren----------------------------------------
function standardize(x)
μ = mean(x; dims = 2)
σ = std(x; dims = 2)
z = (x .- μ) ./ σ
return z, μ, σ
end

function featurize(raw_df, num_train = 20)
raw_df.year = Float64.(year.(raw_df.date))
raw_df.month = Float64.(month.(raw_df.date))
df = combine(groupby(raw_df, [:year, :month]),
:date => (d -> mean(year.(d)) .+ mean(month.(d)) ./ 12),
:meantemp => mean,
:humidity => mean,
:wind_speed => mean,
:meanpressure => mean;
renamecols = false)
t_and_y(df) = df.date', Matrix(select(df, FEATURES))'
t_train, y_train = t_and_y(df[1:num_train, :])
t_test, y_test = t_and_y(df[(num_train + 1):end, :])
t_train, t_mean, t_scale = standardize(t_train)
y_train, y_mean, y_scale = standardize(y_train)
t_test = (t_test .- t_mean) ./ t_scale
y_test = (y_test .- y_mean) ./ y_scale

return (vec(t_train), y_train,
vec(t_test), y_test,
(t_mean, t_scale),
(y_mean, y_scale))
end

function plot_features(t_train, y_train, t_test, y_test)
plt_split = plot(reshape(t_train, :), y_train';
linewidth = 3, colors = 1:4,
xlabel = "Normalized time",
ylabel = "Normalized values",
label = nothing,
title = "Features")
plot!(plt_split, reshape(t_test, :), y_test';
linewidth = 3, linestyle = :dash,
color = [1 2 3 4], label = nothing)
plot!(plt_split, [0], [0]; linewidth = 0,
label = "Train", color = 1)
plot!(plt_split, [0], [0]; linewidth = 0,
linestyle = :dash, label = "Test",
color = 1,
ylims = (-5, 5))
end

t_train, y_train, t_test, y_test, (t_mean, t_scale), (y_mean, y_scale) = featurize(df)
plot_features(t_train, y_train, t_test, y_test)

plot(t_train, y_train[1,:])

#-------------------------------Training des NN------------------------------------------
function neural_ode(t, data_dim)
f = Lux.Chain(Lux.Dense(data_dim => 64, swish), Lux.Dense(64 => 32, swish), Lux.Dense(32 => data_dim))

node = NeuralODE(f, extrema(t), Tsit5(); saveat = t,
abstol = 1e-9, reltol = 1e-9)

rng = Random.default_rng()
p, state = Lux.setup(rng, f)

return node, ComponentArray(p), state
end

function train_one_round(node, p, state, y, opt, maxiters, rng, y0 = [y[1, 1]]; kwargs...)
predict(p) = Array(node(y0, p, state)[1])
loss(p) = sum(abs2, predict(p) .- y)

optf = OptimizationFunction((p, _) -> loss(p), adtype)
optprob = OptimizationProblem(optf, p)
res = solve(optprob, opt; maxiters = maxiters, kwargs...)
res.minimizer, state
end

function train(t, y, obs_grid, maxiters, lr, rng, p = nothing, state = nothing; kwargs...)
log_results(ps, losses) = (p, loss) -> begin
push!(ps, copy(p))
push!(losses, loss)
false
end

ps, losses = ComponentArray[], Float32[]
for k in obs_grid
node, p_new, state_new = neural_ode(t, size(y[1,1], 1))
p === nothing && (p = p_new)                #Löscht Eintrag von p, nimmt jedoch p für train_one_round
state === nothing && (state = state_new)

p, state = train_one_round(node, p, state, y, ADAM(lr), maxiters, rng;
callback = log_results(ps, losses), kwargs...)
end
ps, state, losses
end

rng = MersenneTwister(123)
obs_grid = 4:4:length(t_train) # we train on an increasing amount of the first k obs
maxiters = 150
lr = 5e-3
ps, state, losses = train(t_train, y_train[1,:], obs_grid, maxiters, lr, rng; progress = true)

#--------------------------------Prediction Part------------------------------------------
predict(y0, t, p, state) = begin
node, _, _ = neural_ode(t, length(y0))
Array(node(y0, p, state)[1])
end

t_grid = collect(range(minimum(t_train), maximum(t_train); length = 500))
y_pred = predict([y_train[1, 1]], t_grid, ps[end], state)

plot(t_grid, y_pred[1,:], label = "Prediction")
plot!(t_train, y_train[1,:], label = "Data")
``````

Does the network need the connections between the 4 meteorological aspects in order to learn the course of a single aspect? I know this is a difficult question but maybe someone knows an answer.

Ok sorry for the spam but now i got the solution. It seems to me that the network needs to be connectet with two inputs (time) and two outputs (mathematical function, time) to find a correlation. Now it works.

``````using Random, Dates, Optimization, ComponentArrays, Lux, OptimizationOptimisers, DiffEqFlux,
OrdinaryDiffEq, CSV, DataFrames, Dates, Statistics, Plots, DataDeps
using Flux

#----------------------------------Datenextraktion--------------------------------------------------
time = 0:1:11
t = collect(range(extrema(time)..., length=length(time)))
tspan = (Float32.(t[1]), Float32.(t[end]))
y = exp.(0.05*t).*sin.(t)
data = hcat(y, t)'

u0 = [Float32(data[1])];

#-------------------------------Training des NN------------------------------------------
function neural_ode(t, data_dim)
f = Lux.Chain(Lux.Dense(data_dim => 32, swish), Lux.Dense(32 => 20, swish), Lux.Dense(20 => data_dim))

node = NeuralODE(f, extrema(t), Tsit5(); saveat = t,
abstol = 1e-9, reltol = 1e-9)

rng = Random.default_rng()
p, state = Lux.setup(rng, f)

return node, ComponentArray(p), state
end

function train_one_round(node, p, state, y, opt, maxiters, rng, y0 = y[:, 1]; kwargs...)
predict(p) = Array(node(y0, p, state)[1])
loss(p) = sum(abs2, predict(p) .- y)

optf = OptimizationFunction((p, _) -> loss(p), adtype)
optprob = OptimizationProblem(optf, p)
res = solve(optprob, opt; maxiters = maxiters, kwargs...)
res.minimizer, state
end

function train(t, y, obs_grid, maxiters, lr, rng, p = nothing, state = nothing; kwargs...)
log_results(ps, losses) = (p, loss) -> begin
push!(ps, copy(p))
push!(losses, loss)
false
end

ps, losses = ComponentArray[], Float32[]
for k in obs_grid
node, p_new, state_new = neural_ode(t, size(y, 1))
p === nothing && (p = p_new)
state === nothing && (state = state_new)

p, state = train_one_round(node, p, state, y, ADAM(lr), maxiters, rng;
callback = log_results(ps, losses), kwargs...)
end
ps, state, losses
end

rng = MersenneTwister(123)
obs_grid = 4:4:length(time) # we train on an increasing amount of the first k obs
maxiters = 500
lr = 5e-3
ps, state, losses = train(time, data, obs_grid, maxiters, lr, rng; progress = true)

plot(losses, yaxis = :log)

#-------------------Prediction Part--------------------------------

predict(y0, t, p, state) = begin
node, _, _ = neural_ode(t, length(y0))
Array(node(y0, p, state)[1])
end

t_grid = Float32.(collect(range(minimum(time), maximum(time); length = 500)))
y_pred = predict(data[:,1], t_grid, ps[end], state)

plot(time, data[1,:], label="Data", xlabel="time", ylabel="data", linewidth=2)
plot!(t_grid, y_pred[1,:], label="Prediction", linestyle=:dash, linewidth=2)

``````