Saving StructArray with Arrow.jl fails

General Usage
,
1

I am trying to save my flight data with Arrow.jl.

The code looks like this:

using Rotations, StaticArrays, StructArrays, Arrow

const MyFloat = Float32
const SEGMENTS = 7                    # number of tether segments
const SAMPLE_FREQ = 20                # sample frequency in Hz
const DATA_PATH = "."            # path for log files and other data

struct SysState
    time::Float64                     # time since launch in seconds
    orient::Quat                      # orientation of the kite
    X::MVector{SEGMENTS+1, MyFloat}   # vector of particle positions in x
    Y::MVector{SEGMENTS+1, MyFloat}   # vector of particle positions in y
    Z::MVector{SEGMENTS+1, MyFloat}   # vector of particle positions in z
end 

# extended SysState containing derived values for plotting
struct ExtSysState
    time::Float64                     # time since launch in seconds
    orient::Quat                      # orientation of the kite
    X::MVector{SEGMENTS+1, MyFloat}   # vector of particle positions in x
    Y::MVector{SEGMENTS+1, MyFloat}   # vector of particle positions in y
    Z::MVector{SEGMENTS+1, MyFloat}   # vector of particle positions in z
    x::MyFloat                        # kite position in x
    y::MyFloat                        # kite position in y
    z::MyFloat                        # kite position in z
end 

struct FlightLog
    name::String
    log_3d::Vector{SysState}          # vector of structs
    log_2d::StructArray{ExtSysState}  # struct of vectors, derived from log_3d
end

# create a demo state with a given height and time
function demo_state(height=6.0, time=0.0)
    a = 10
    X = range(0, stop=10, length=SEGMENTS+1)
    Y = zeros(length(X))
    Z = (a .* cosh.(X./a) .- a) * height/ 5.430806 
    orient = UnitQuaternion(1.0,0,0,0)
    return SysState(time, orient, X, Y, Z)
end

# create a demo flight log for 3d replay with given name [String] and duration [s]
function demo_log3d(name="Test flight"; duration=10)
    max_height = 6.0
    steps   = Int(duration * SAMPLE_FREQ) + 1
    log_3d = Vector{SysState}(undef, steps)
    for i in range(0, length=steps)
        log_3d[i+1] = demo_state(max_height * i/steps, i/SAMPLE_FREQ)
    end
    return log_3d
end

# convert vector of structs to struct of vectors for easy plotting in 2d
function vos2sov(log::Vector)
    steps=length(log)
    time_vec = Vector{Float64}(undef, steps)
    orient_vec = Vector{Quat}(undef, steps)
    X_vec = Vector{MVector{SEGMENTS+1, MyFloat}}(undef, steps)
    Y_vec = Vector{MVector{SEGMENTS+1, MyFloat}}(undef, steps)
    Z_vec = Vector{MVector{SEGMENTS+1, MyFloat}}(undef, steps)
    x = Vector{MyFloat}(undef, steps)
    y = Vector{MyFloat}(undef, steps)
    z = Vector{MyFloat}(undef, steps)
    for i in range(1, length=steps)
        state=log[i]
        time_vec[i] = state.time
        orient_vec[i] = state.orient
        X_vec[i] = state.X
        Y_vec[i] = state.Y
        Z_vec[i] = state.Z
        x[i] = state.X[end]
        y[i] = state.Y[end]
        z[i] = state.Z[end]
    end
    return StructArray{ExtSysState}((time_vec, orient_vec, X_vec, Y_vec, Z_vec, x, y, z))
end

function demo_log(name="Test_flight"; duration=10)
    log_3d = demo_log3d(name, duration=duration)
    log_2d = vos2sov(log_3d)
    return FlightLog(name, log_3d, log_2d)
end

function save_log(log::FlightLog)
    filename=joinpath(DATA_PATH, log.name) * ".arrow"
    println(filename)
    Arrow.write(filename, log.log_2d)
    println(filename)
end

flight_log=demo_log()
save_log(flight_log)

If I include it I get the following error message:

julia> include("ArrowMinimal.jl")
./Test_flight.arrow
ERROR: LoadError: ArgumentError: type does not have a definite number of fields
Stacktrace:
  [1] fieldcount
    @ ./reflection.jl:729 [inlined]
  [2] arrowvector(::Arrow.ArrowTypes.StructType, x::Arrow.ToList{Any, false, UnitQuaternion, Int32}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Dict{String, String}; kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Bool}, NTuple{5, Symbol}, NamedTuple{(:dictencode, :lareglists, :compression, :denseunions, :dictencodenested), Tuple{Bool, Bool, Nothing, Bool, Bool}}})
    @ Arrow ~/.julia/packages/Arrow/Re9EM/src/arraytypes/struct.jl:89
  [3] arrowvector(::Type{Any}, x::Arrow.ToList{Any, false, UnitQuaternion, Int32}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Bool}, NTuple{5, Symbol}, NamedTuple{(:dictencode, :lareglists, :compression, :denseunions, :dictencodenested), Tuple{Bool, Bool, Nothing, Bool, Bool}}})
    @ Arrow ~/.julia/packages/Arrow/Re9EM/src/arraytypes/arraytypes.jl:85
  [4] arrowvector(x::Arrow.ToList{Any, false, UnitQuaternion, Int32}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:lareglists, :compression, :denseunions, :dictencodenested), Tuple{Bool, Nothing, Bool, Bool}}})
    @ Arrow ~/.julia/packages/Arrow/Re9EM/src/arraytypes/arraytypes.jl:58
  [5] arrowvector(::Arrow.ArrowTypes.ListType, x::Vector{UnitQuaternion}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; largelists::Bool, kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:dictencode, :compression, :denseunions, :dictencodenested), Tuple{Bool, Nothing, Bool, Bool}}})
    @ Arrow ~/.julia/packages/Arrow/Re9EM/src/arraytypes/list.jl:191
  [6] arrowvector(::Type{UnitQuaternion}, x::Vector{UnitQuaternion}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Bool}, NTuple{5, Symbol}, NamedTuple{(:dictencode, :compression, :largelists, :denseunions, :dictencodenested), Tuple{Bool, Nothing, Bool, Bool, Bool}}})
    @ Arrow ~/.julia/packages/Arrow/Re9EM/src/arraytypes/arraytypes.jl:90
  [7] arrowvector(x::Vector{UnitQuaternion}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, kw::Base.Iterators.Pairs{Symbol, Union{Nothing, Bool}, NTuple{4, Symbol}, NamedTuple{(:compression, :largelists, :denseunions, :dictencodenested), Tuple{Nothing, Bool, Bool, Bool}}})
    @ Arrow ~/.julia/packages/Arrow/Re9EM/src/arraytypes/arraytypes.jl:58
  [8] toarrowvector(x::Vector{UnitQuaternion}, i::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; compression::Nothing, kw::Base.Iterators.Pairs{Symbol, Bool, NTuple{4, Symbol}, NamedTuple{(:largelists, :denseunions, :dictencode, :dictencodenested), NTuple{4, Bool}}})
    @ Arrow ~/.julia/packages/Arrow/Re9EM/src/arraytypes/arraytypes.jl:36
  [9] (::Arrow.var"#109#110"{Dict{Int64, Any}, Bool, Nothing, Bool, Bool, Bool, Vector{Arrow.DictEncoding}, Vector{Type}, Vector{Any}})(col::Vector{UnitQuaternion}, i::Int64, nm::Symbol)
    @ Arrow ~/.julia/packages/Arrow/Re9EM/src/write.jl:202
 [10] eachcolumn
    @ ~/.julia/packages/Tables/nGOci/src/utils.jl:70 [inlined]
 [11] toarrowtable(x::StructVector{ExtSysState, NamedTuple{(:time, :orient, :X, :Y, :Z, :x, :y, :z), Tuple{Vector{Float64}, Vector{UnitQuaternion}, Vector{MVector{8, Float32}}, Vector{MVector{8, Float32}}, Vector{MVector{8, Float32}}, Vector{Float32}, Vector{Float32}, Vector{Float32}}}, Int64}, dictencodings::Dict{Int64, Any}, largelists::Bool, compress::Nothing, denseunions::Bool, dictencode::Bool, dictencodenested::Bool)
    @ Arrow ~/.julia/packages/Arrow/Re9EM/src/write.jl:201
 [12] macro expansion
    @ ~/.julia/packages/Arrow/Re9EM/src/write.jl:111 [inlined]
 [13] macro expansion
    @ ./task.jl:382 [inlined]
 [14] write(io::IOStream, source::StructVector{ExtSysState, NamedTuple{(:time, :orient, :X, :Y, :Z, :x, :y, :z), Tuple{Vector{Float64}, Vector{UnitQuaternion}, Vector{MVector{8, Float32}}, Vector{MVector{8, Float32}}, Vector{MVector{8, Float32}}, Vector{Float32}, Vector{Float32}, Vector{Float32}}}, Int64}, writetofile::Bool, largelists::Bool, compress::Nothing, denseunions::Bool, dictencode::Bool, dictencodenested::Bool, alignment::Int64)
    @ Arrow ~/.julia/packages/Arrow/Re9EM/src/write.jl:108
 [15] #100
    @ ~/.julia/packages/Arrow/Re9EM/src/write.jl:77 [inlined]
 [16] open(::Arrow.var"#100#101"{Bool, Nothing, Bool, Bool, Bool, Int64, StructVector{ExtSysState, NamedTuple{(:time, :orient, :X, :Y, :Z, :x, :y, :z), Tuple{Vector{Float64}, Vector{UnitQuaternion}, Vector{MVector{8, Float32}}, Vector{MVector{8, Float32}}, Vector{MVector{8, Float32}}, Vector{Float32}, Vector{Float32}, Vector{Float32}}}, Int64}}, ::String, ::Vararg{String, N} where N; kwargs::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Base ./io.jl:330
 [17] open(::Function, ::String, ::String)
    @ Base ./io.jl:328
 [18] #write#99
    @ ~/.julia/packages/Arrow/Re9EM/src/write.jl:76 [inlined]
 [19] write
    @ ~/.julia/packages/Arrow/Re9EM/src/write.jl:76 [inlined]
 [20] save_log(log::FlightLog)
    @ Main ~/repos/KiteViewer/src/ArrowMinimal.jl:112
 [21] top-level scope
    @ ~/repos/KiteViewer/src/ArrowMinimal.jl:117
 [22] include(fname::String)
    @ Base.MainInclude ./client.jl:444
 [23] top-level scope
    @ REPL[2]:1
in expression starting at /home/ufechner/repos/KiteViewer/src/ArrowMinimal.jl:117

julia>

Any idea?

2

If I remove all fields of type MVector from the structs it works…

The error message is wrong in the sense that MVectors DO have a definite number of fields…

Any idea for a workaround?

3

A very simple example of saving structs that contain a float, a quaternion and an MVector works:

using Arrow, Rotations, StaticArrays

const SEGMENTS = 7
const MyFloat  = Float32

struct SysState
    time::Float64                     # time since launch in seconds
    orient::UnitQuaternion{MyFloat}   # orientation of the kite
    X::MVector{SEGMENTS+1, MyFloat}
end 

Arrow.ArrowTypes.registertype!(SysState, SysState)

orient = UnitQuaternion(1.0,0,0,0)
X = range(0, stop=10, length=8)

table = (col1=[SysState(1, orient, X), SysState(2, orient, X)],)
io = IOBuffer()
Arrow.write(io, table)
seekstart(io)
table2 = Arrow.Table(io)

Still need to figure out what is wrong with the original example.

4

I could fix the original example, it is working now:

using Rotations, StaticArrays, StructArrays, Arrow

const MyFloat = Float32
const SEGMENTS = 7                    # number of tether segments
const SAMPLE_FREQ = 20                # sample frequency in Hz
const DATA_PATH = "."                 # path for log files and other data

struct SysState
    time::Float64                     # time since launch in seconds
    orient::UnitQuaternion{Float32}   # orientation of the kite
    X::MVector{SEGMENTS+1, MyFloat}   # vector of particle positions in x
    Y::MVector{SEGMENTS+1, MyFloat}   # vector of particle positions in y
    Z::MVector{SEGMENTS+1, MyFloat}   # vector of particle positions in z
end 

# extended SysState containing derived values for plotting
struct ExtSysState
    time::Float64                     # time since launch in seconds
    orient::UnitQuaternion{Float32}   # orientation of the kite
    X::MVector{SEGMENTS+1, MyFloat}   # vector of particle positions in x
    Y::MVector{SEGMENTS+1, MyFloat}   # vector of particle positions in y
    Z::MVector{SEGMENTS+1, MyFloat}   # vector of particle positions in z
    x::MyFloat                        # kite position in x
    y::MyFloat                        # kite position in y
    z::MyFloat                        # kite position in z
end 

Arrow.ArrowTypes.registertype!(ExtSysState, ExtSysState)

struct FlightLog
    name::String
    log_3d::Vector{SysState}          # vector of structs
    log_2d::StructArray{ExtSysState}  # struct of vectors, derived from log_3d
end

# create a demo state with a given height and time
function demo_state(height=6.0, time=0.0)
    a = 10
    X = range(0, stop=10, length=SEGMENTS+1)
    Y = zeros(length(X))
    Z = (a .* cosh.(X./a) .- a) * height/ 5.430806 
    orient = UnitQuaternion(1.0,0,0,0)
    return SysState(time, orient, X, Y, Z)
end

# create a demo flight log for 3d replay with given name [String] and duration [s]
function demo_log3d(name="Test flight"; duration=10)
    max_height = 6.0
    steps   = Int(duration * SAMPLE_FREQ) + 1
    log_3d = Vector{SysState}(undef, steps)
    for i in range(0, length=steps)
        log_3d[i+1] = demo_state(max_height * i/steps, i/SAMPLE_FREQ)
    end
    return log_3d
end

# convert vector of structs to struct of vectors for easy plotting in 2d
function vos2sov(log::Vector)
    steps=length(log)
    time_vec = Vector{Float64}(undef, steps)
    orient_vec = Vector{Quat}(undef, steps)
    X_vec = Vector{MVector{SEGMENTS+1, MyFloat}}(undef, steps)
    Y_vec = Vector{MVector{SEGMENTS+1, MyFloat}}(undef, steps)
    Z_vec = Vector{MVector{SEGMENTS+1, MyFloat}}(undef, steps)
    x = Vector{MyFloat}(undef, steps)
    y = Vector{MyFloat}(undef, steps)
    z = Vector{MyFloat}(undef, steps)
    for i in range(1, length=steps)
        state=log[i]
        time_vec[i] = state.time
        orient_vec[i] = state.orient
        X_vec[i] = state.X
        Y_vec[i] = state.Y
        Z_vec[i] = state.Z
        x[i] = state.X[end]
        y[i] = state.Y[end]
        z[i] = state.Z[end]
    end
    return StructArray{ExtSysState}((time_vec, orient_vec, X_vec, Y_vec, Z_vec, x, y, z))
end

function demo_log(name="Test_flight"; duration=10)
    log_3d = demo_log3d(name, duration=duration)
    log_2d = vos2sov(log_3d)
    return FlightLog(name, log_3d, log_2d)
end

function save_log(flight_log)
    table = (col1=flight_log.log_3d,)
    filename=joinpath(DATA_PATH, flight_log.name) * ".arrow"
    Arrow.write(filename, table)
end

function main()
    flight_log=demo_log()
    save_log(flight_log)
end

Not so sure why, but I save now a named tuple instead of a StructArray, that is good enough for me.

1 Like