NeuralODE - Array has no field layer

General Usage
1

Hi,

I’m new here and have a question about extrapolating data using neural ordinary differential equations. I am in the process of training a mathematical function in steps of four. Training the neural network seems to work. The parameters of the network are adjusted. However, I am not able to plot a prediction with the trained network.

Here is the code:

using Random, Dates, Optimization, ComponentArrays, Lux, OptimizationOptimisers, DiffEqFlux,
    OrdinaryDiffEq, CSV, DataFrames, Dates, Statistics, Plots, DataDeps
using Flux

#----------------------------------Data Extraction--------------------------------------------------
time = 0:1:15
t = collect(range(extrema(time)..., length=length(time)))
tspan = (Float32.(t[1]), Float32.(t[end]))
#tspan = (Float32.(t_scale[1]), Float32.(t_scale[end]));
data = sin.(t).*exp.(-0.05*t)
plot(t, data)

u0 = [Float32(data[1])];

#-------------------------------Training NN------------------------------------------
function neural_ode(t, data_dim)
    f = Lux.Chain(Lux.Dense(data_dim => 64, swish), Lux.Dense(64 => 32, swish), Lux.Dense(32 => data_dim))

    node = NeuralODE(f, extrema(t), Tsit5(); saveat = t,
        abstol = 1e-9, reltol = 1e-9)

    rng = Random.default_rng()
    p, state = Lux.setup(rng, f)

    return node, ComponentArray(p), state
end

function train_one_round(node, p, state, y, opt, maxiters, rng, y0 = y[:, 1]; kwargs...)
    predict(p) = Array(node(y0, p, state)[1])
    loss(p) = sum(abs2, predict(p) .- y)

    adtype = Optimization.AutoZygote()
    optf = OptimizationFunction((p, _) -> loss(p), adtype)
    optprob = OptimizationProblem(optf, p)
    res = solve(optprob, opt; maxiters = maxiters, kwargs...)
    res.minimizer, state
end

function train(t, y, obs_grid, maxiters, lr, rng, p = nothing, state = nothing; kwargs...)
    log_results(ps, losses) = (p, loss) -> begin
        push!(ps, copy(p))
        push!(losses, loss)
        false
    end

    ps, losses = ComponentArray[], Float32[]
    for k in obs_grid
        node, p_new, state_new = neural_ode(t, size(y, 1))
        p === nothing && (p = p_new)
        state === nothing && (state = state_new)

        p, state = train_one_round(node, p, state, y, ADAM(lr), maxiters, rng;
            callback = log_results(ps, losses), kwargs...)
    end
    ps, state, losses
end

rng = MersenneTwister(123)
obs_grid = 4:4:length(time) # we train on an increasing amount of the first k obs
maxiters = 150
lr = 5e-3
ps, state, losses = train(time, data, obs_grid, maxiters, lr, rng; progress = true)

#-------------------Prediction Part--------------------------------
t_grid = Float32.(collect(range(minimum(time), maximum(time); length = 500)))
predict(y0, t, p, state) = begin
    node, _, _ = neural_ode(t, length(y0))
    Array(node(y0, p, state)[1])
end

y_pred = predict(u0, t_grid, ps, state)

plot(time, data, label="Originaldaten", xlabel="Zeit", ylabel="Daten", linewidth=2)
plot!(time, y_pred, label="Vorhersagen", linestyle=:dash, linewidth=2)
legend()

I’m getting an error at this point:

predict(y0, t, p, state) = begin
    node, _, _ = neural_ode(t, length(y0))
    Array(node(y0, p, state)[1])
end

y_pred = predict(Float64(data[1]), t_grid, ps, state)

ERROR: type Array has no field layer_1

Does anyone know how i can solve this problem?

2

It means the parameters you’re passing to the neural network are of type Array and not of type ComponentArray. Track down where the conversion was made or missing.

3

Thank you for your answer! I forgot to use the last parameter set of the training for the prediction. Here ist the solution:

predict(y0, t, p, state) = begin
    node, _, _ = neural_ode(t, length(y0))
    Array(node(y0, p, state)[1])
end

y_pred = predict(Float32(data[1]), t_grid, ps[end], state)

Now I’m getting the next error at the same point:
ERROR: DimensionMismatch: second dimension of A, 16, does not match length
of x, 1

4

Apparently the prediction has a problem with the dimension of the vector “time” but I don’t know why.

5

I’m not 100% sure what your follow up question is.

6

Is it possible that the NODE is not able to learn a simple mathematical function?

I tried one code which uses 4 different time series of a weather forecast which means that the network hast 4 inputs of time and 4 outputs (Temperature, Humidity,Velocity, Pressure) of the meteorological aspects. This code works verry well.

using Random, Dates, Optimization, ComponentArrays, Lux, OptimizationOptimisers, DiffEqFlux,
    OrdinaryDiffEq, CSV, DataFrames, Dates, Statistics, Plots, DataDeps

#----------------------------------Datenextraktion--------------------------------------------------
function download_data(data_url = "https://raw.githubusercontent.com/SebastianCallh/neural-ode-weather-forecast/master/data/",
        data_local_path = "./delhi")
    function load(file_name)
        data_dep = DataDep("delhi/train", "", "$data_url/$file_name")
        Base.download(data_dep, data_local_path; i_accept_the_terms_of_use = true)
        CSV.read(joinpath(data_local_path, file_name), DataFrame)
    end

    train_df = load("DailyDelhiClimateTrain.csv")
    test_df = load("DailyDelhiClimateTest.csv")
    return vcat(train_df, test_df)
end

df = download_data()

#----------------------------------Daten ordnen--------------------------------------------------------------
FEATURES = [:meantemp, :humidity, :wind_speed, :meanpressure]
UNITS = ["Celsius", "g/m³ of water", "km/h", "hPa"]
FEATURE_NAMES = ["Mean temperature", "Humidity", "Wind speed", "Mean pressure"]

function plot_data(df)
    plots = map(enumerate(zip(FEATURES, FEATURE_NAMES, UNITS))) do (i, (f, n, u))
        plot(df[:, :date], df[:, f]; title = n, label = nothing,
            ylabel = u, size = (800, 600), color = i)
    end

    n = length(plots)
    plot(plots...; layout = (Int(n / 2), Int(n / 2)))
end

plot_data(df)

#------------------------------Daten standardisieren----------------------------------------
function standardize(x)
    μ = mean(x; dims = 2)
    σ = std(x; dims = 2)
    z = (x .- μ) ./ σ
    return z, μ, σ
end

function featurize(raw_df, num_train = 20)
    raw_df.year = Float64.(year.(raw_df.date))
    raw_df.month = Float64.(month.(raw_df.date))
    df = combine(groupby(raw_df, [:year, :month]),
        :date => (d -> mean(year.(d)) .+ mean(month.(d)) ./ 12),
        :meantemp => mean,
        :humidity => mean,
        :wind_speed => mean,
        :meanpressure => mean;
        renamecols = false)
    t_and_y(df) = df.date', Matrix(select(df, FEATURES))'
    t_train, y_train = t_and_y(df[1:num_train, :])
    t_test, y_test = t_and_y(df[(num_train + 1):end, :])
    t_train, t_mean, t_scale = standardize(t_train)
    y_train, y_mean, y_scale = standardize(y_train)
    t_test = (t_test .- t_mean) ./ t_scale
    y_test = (y_test .- y_mean) ./ y_scale

    return (vec(t_train), y_train,
        vec(t_test), y_test,
        (t_mean, t_scale),
        (y_mean, y_scale))
end

function plot_features(t_train, y_train, t_test, y_test)
    plt_split = plot(reshape(t_train, :), y_train';
        linewidth = 3, colors = 1:4,
        xlabel = "Normalized time",
        ylabel = "Normalized values",
        label = nothing,
        title = "Features")
    plot!(plt_split, reshape(t_test, :), y_test';
        linewidth = 3, linestyle = :dash,
        color = [1 2 3 4], label = nothing)
    plot!(plt_split, [0], [0]; linewidth = 0,
        label = "Train", color = 1)
    plot!(plt_split, [0], [0]; linewidth = 0,
        linestyle = :dash, label = "Test",
        color = 1,
        ylims = (-5, 5))
end

t_train, y_train, t_test, y_test, (t_mean, t_scale), (y_mean, y_scale) = featurize(df)
plot_features(t_train, y_train, t_test, y_test)

#-------------------------------Training des NN------------------------------------------
function neural_ode(t, data_dim)
    f = Lux.Chain(Lux.Dense(data_dim => 64, swish), Lux.Dense(64 => 32, swish), Lux.Dense(32 => data_dim))

    node = NeuralODE(f, extrema(t), Tsit5(); saveat = t,
        abstol = 1e-9, reltol = 1e-9)

    rng = Random.default_rng()
    p, state = Lux.setup(rng, f)

    return node, ComponentArray(p), state
end

function train_one_round(node, p, state, y, opt, maxiters, rng, y0 = y[:, 1]; kwargs...)
    predict(p) = Array(node(y0, p, state)[1])
    loss(p) = sum(abs2, predict(p) .- y)

    adtype = Optimization.AutoZygote()
    optf = OptimizationFunction((p, _) -> loss(p), adtype)
    optprob = OptimizationProblem(optf, p)
    res = solve(optprob, opt; maxiters = maxiters, kwargs...)
    res.minimizer, state
end

function train(t, y, obs_grid, maxiters, lr, rng, p = nothing, state = nothing; kwargs...)
    log_results(ps, losses) = (p, loss) -> begin
        push!(ps, copy(p))
        push!(losses, loss)
        false
    end

    ps, losses = ComponentArray[], Float32[]
    for k in obs_grid
        node, p_new, state_new = neural_ode(t, size(y, 1))
        p === nothing && (p = p_new)                #Löscht Eintrag von p, nimmt jedoch p für train_one_round
        state === nothing && (state = state_new)

        p, state = train_one_round(node, p, state, y, ADAM(lr), maxiters, rng;
            callback = log_results(ps, losses), kwargs...)
    end
    ps, state, losses
end

rng = MersenneTwister(123)
obs_grid = 4:4:length(t_train) # we train on an increasing amount of the first k obs
maxiters = 150
lr = 5e-3
ps, state, losses = train(t_train, y_train, obs_grid, maxiters, lr, rng; progress = true)

#------------------------------Prediction Part-------------------------------------------
predict(y0, t, p, state) = begin
    node, _, _ = neural_ode(t, length(y0))
    Array(node(y0, p, state)[1])
end

node, _, _ = neural_ode(t, length(y_train[:, 1]))
y_pred = Array(node(y_train[:, 1], ps[end], state)[1])

t_grid = collect(range(minimum(t_train), maximum(t_test); length = 500))
y_pred = predict(y_train[:, 1], t_grid, ps[end], state)

plot(t_grid, y_pred[1,:], label = "Prediction")
plot!(t_train, y_train[1,:], label = "Data")

Prediction_4InputsOutputs

Then I reduced the Inputs and Outputs to 1 (Only Temperature) and trained again. Now it does’nt work anymore.

using Random, Dates, Optimization, ComponentArrays, Lux, OptimizationOptimisers, DiffEqFlux,
    OrdinaryDiffEq, CSV, DataFrames, Dates, Statistics, Plots, DataDeps

#----------------------------------Datenextraktion--------------------------------------------------
function download_data(data_url = "https://raw.githubusercontent.com/SebastianCallh/neural-ode-weather-forecast/master/data/",
        data_local_path = "./delhi")
    function load(file_name)
        data_dep = DataDep("delhi/train", "", "$data_url/$file_name")
        Base.download(data_dep, data_local_path; i_accept_the_terms_of_use = true)
        CSV.read(joinpath(data_local_path, file_name), DataFrame)
    end

    train_df = load("DailyDelhiClimateTrain.csv")
    test_df = load("DailyDelhiClimateTest.csv")
    return vcat(train_df, test_df)
end

df = download_data()

#----------------------------------Daten ordnen--------------------------------------------------------------
FEATURES = [:meantemp, :humidity, :wind_speed, :meanpressure]
UNITS = ["Celsius", "g/m³ of water", "km/h", "hPa"]
FEATURE_NAMES = ["Mean temperature", "Humidity", "Wind speed", "Mean pressure"]

function plot_data(df)
    plots = map(enumerate(zip(FEATURES, FEATURE_NAMES, UNITS))) do (i, (f, n, u))
        plot(df[:, :date], df[:, f]; title = n, label = nothing,
            ylabel = u, size = (800, 600), color = i)
    end

    n = length(plots)
    plot(plots...; layout = (Int(n / 2), Int(n / 2)))
end

plot_data(df)

#------------------------------Daten standardisieren----------------------------------------
function standardize(x)
    μ = mean(x; dims = 2)
    σ = std(x; dims = 2)
    z = (x .- μ) ./ σ
    return z, μ, σ
end

function featurize(raw_df, num_train = 20)
    raw_df.year = Float64.(year.(raw_df.date))
    raw_df.month = Float64.(month.(raw_df.date))
    df = combine(groupby(raw_df, [:year, :month]),
        :date => (d -> mean(year.(d)) .+ mean(month.(d)) ./ 12),
        :meantemp => mean,
        :humidity => mean,
        :wind_speed => mean,
        :meanpressure => mean;
        renamecols = false)
    t_and_y(df) = df.date', Matrix(select(df, FEATURES))'
    t_train, y_train = t_and_y(df[1:num_train, :])
    t_test, y_test = t_and_y(df[(num_train + 1):end, :])
    t_train, t_mean, t_scale = standardize(t_train)
    y_train, y_mean, y_scale = standardize(y_train)
    t_test = (t_test .- t_mean) ./ t_scale
    y_test = (y_test .- y_mean) ./ y_scale

    return (vec(t_train), y_train,
        vec(t_test), y_test,
        (t_mean, t_scale),
        (y_mean, y_scale))
end

function plot_features(t_train, y_train, t_test, y_test)
    plt_split = plot(reshape(t_train, :), y_train';
        linewidth = 3, colors = 1:4,
        xlabel = "Normalized time",
        ylabel = "Normalized values",
        label = nothing,
        title = "Features")
    plot!(plt_split, reshape(t_test, :), y_test';
        linewidth = 3, linestyle = :dash,
        color = [1 2 3 4], label = nothing)
    plot!(plt_split, [0], [0]; linewidth = 0,
        label = "Train", color = 1)
    plot!(plt_split, [0], [0]; linewidth = 0,
        linestyle = :dash, label = "Test",
        color = 1,
        ylims = (-5, 5))
end

t_train, y_train, t_test, y_test, (t_mean, t_scale), (y_mean, y_scale) = featurize(df)
plot_features(t_train, y_train, t_test, y_test)

plot(t_train, y_train[1,:])

#-------------------------------Training des NN------------------------------------------
function neural_ode(t, data_dim)
    f = Lux.Chain(Lux.Dense(data_dim => 64, swish), Lux.Dense(64 => 32, swish), Lux.Dense(32 => data_dim))

    node = NeuralODE(f, extrema(t), Tsit5(); saveat = t,
        abstol = 1e-9, reltol = 1e-9)

    rng = Random.default_rng()
    p, state = Lux.setup(rng, f)

    return node, ComponentArray(p), state
end

function train_one_round(node, p, state, y, opt, maxiters, rng, y0 = [y[1, 1]]; kwargs...)
    predict(p) = Array(node(y0, p, state)[1])
    loss(p) = sum(abs2, predict(p) .- y)

    adtype = Optimization.AutoZygote()
    optf = OptimizationFunction((p, _) -> loss(p), adtype)
    optprob = OptimizationProblem(optf, p)
    res = solve(optprob, opt; maxiters = maxiters, kwargs...)
    res.minimizer, state
end

function train(t, y, obs_grid, maxiters, lr, rng, p = nothing, state = nothing; kwargs...)
    log_results(ps, losses) = (p, loss) -> begin
        push!(ps, copy(p))
        push!(losses, loss)
        false
    end

    ps, losses = ComponentArray[], Float32[]
    for k in obs_grid
        node, p_new, state_new = neural_ode(t, size(y[1,1], 1))
        p === nothing && (p = p_new)                #Löscht Eintrag von p, nimmt jedoch p für train_one_round
        state === nothing && (state = state_new)

        p, state = train_one_round(node, p, state, y, ADAM(lr), maxiters, rng;
            callback = log_results(ps, losses), kwargs...)
    end
    ps, state, losses
end

rng = MersenneTwister(123)
obs_grid = 4:4:length(t_train) # we train on an increasing amount of the first k obs
maxiters = 150
lr = 5e-3
ps, state, losses = train(t_train, y_train[1,:], obs_grid, maxiters, lr, rng; progress = true)

#--------------------------------Prediction Part------------------------------------------
predict(y0, t, p, state) = begin
    node, _, _ = neural_ode(t, length(y0))
    Array(node(y0, p, state)[1])
end

t_grid = collect(range(minimum(t_train), maximum(t_train); length = 500))
y_pred = predict([y_train[1, 1]], t_grid, ps[end], state)

plot(t_grid, y_pred[1,:], label = "Prediction")
plot!(t_train, y_train[1,:], label = "Data")

Prediction_1InputOutput

Does the network need the connections between the 4 meteorological aspects in order to learn the course of a single aspect? I know this is a difficult question but maybe someone knows an answer.

7

Ok sorry for the spam but now i got the solution. It seems to me that the network needs to be connectet with two inputs (time) and two outputs (mathematical function, time) to find a correlation. Now it works.

using Random, Dates, Optimization, ComponentArrays, Lux, OptimizationOptimisers, DiffEqFlux,
    OrdinaryDiffEq, CSV, DataFrames, Dates, Statistics, Plots, DataDeps
using Flux

#----------------------------------Datenextraktion--------------------------------------------------
time = 0:1:11
t = collect(range(extrema(time)..., length=length(time)))
tspan = (Float32.(t[1]), Float32.(t[end]))
y = exp.(0.05*t).*sin.(t)
data = hcat(y, t)'

u0 = [Float32(data[1])];

#-------------------------------Training des NN------------------------------------------
function neural_ode(t, data_dim)
    f = Lux.Chain(Lux.Dense(data_dim => 32, swish), Lux.Dense(32 => 20, swish), Lux.Dense(20 => data_dim))

    node = NeuralODE(f, extrema(t), Tsit5(); saveat = t,
        abstol = 1e-9, reltol = 1e-9)

    rng = Random.default_rng()
    p, state = Lux.setup(rng, f)

    return node, ComponentArray(p), state
end

function train_one_round(node, p, state, y, opt, maxiters, rng, y0 = y[:, 1]; kwargs...)
    predict(p) = Array(node(y0, p, state)[1])
    loss(p) = sum(abs2, predict(p) .- y)

    adtype = Optimization.AutoZygote()
    optf = OptimizationFunction((p, _) -> loss(p), adtype)
    optprob = OptimizationProblem(optf, p)
    res = solve(optprob, opt; maxiters = maxiters, kwargs...)
    res.minimizer, state
end

function train(t, y, obs_grid, maxiters, lr, rng, p = nothing, state = nothing; kwargs...)
    log_results(ps, losses) = (p, loss) -> begin
        push!(ps, copy(p))
        push!(losses, loss)
        false
    end

    ps, losses = ComponentArray[], Float32[]
    for k in obs_grid
        node, p_new, state_new = neural_ode(t, size(y, 1))
        p === nothing && (p = p_new)
        state === nothing && (state = state_new)

        p, state = train_one_round(node, p, state, y, ADAM(lr), maxiters, rng;
            callback = log_results(ps, losses), kwargs...)
    end
    ps, state, losses
end

rng = MersenneTwister(123)
obs_grid = 4:4:length(time) # we train on an increasing amount of the first k obs
maxiters = 500
lr = 5e-3
ps, state, losses = train(time, data, obs_grid, maxiters, lr, rng; progress = true)

plot(losses, yaxis = :log)

#-------------------Prediction Part--------------------------------

predict(y0, t, p, state) = begin
    node, _, _ = neural_ode(t, length(y0))
    Array(node(y0, p, state)[1])
end

t_grid = Float32.(collect(range(minimum(time), maximum(time); length = 500)))
y_pred = predict(data[:,1], t_grid, ps[end], state)

plot(time, data[1,:], label="Data", xlabel="time", ylabel="data", linewidth=2)
plot!(t_grid, y_pred[1,:], label="Prediction", linestyle=:dash, linewidth=2)

Prediction_MathFunc