Write structure to HDF5 file and read a compound type from HDF5 into Julia structure

sairus7 · 2019-02-01 16:05:04 UTC

How to write a structure in HDF5 file?

using HDF5
mutable struct Foo
    a::Int
    b::Float64
end
vect = [1,2,3,4,5]
foo = Foo(1, 2.0)
foovect = [P(i, i/2) for i=1:5]
path = "/testfile.h5"
h5write(path, "/vect", vect)
h5write(path, "/foo", foo) 
# > MethodError: no method matching write(::HDF5File, ::String, ::Foo)
h5write(path, "/foovec", foovect)
# > MethodError: no method matching write(::HDF5File, ::String, ::Array{P,1})

And how to read a compound type?
If I write some compound type into a hdf5 with c++ (Hello World! into a byte fields named as field0 … field11), then try to read it in Julia, it reads an array of HDF5Compound type:

HDF5.HDF5Compound{12}
data -> (0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21), 
membername -> ("0\xfd\xf6\x11field0", "0\xfd\xf6\x11field1", "0\xfd\xf6\x11field2", "0\xfd\xf6\x11field3", "0\xfd\xf6\x11field4", "0\xfd\xf6\x11field5", "0\xfd\xf6\x11field6", "0\xfd\xf6\x11field7", "0\xfd\xf6\x11field8", "0\xfd\xf6\x11field9", "0\xfd\xf6\x11field10", "0\xfd\xf6\x11field11"), 
membertype -> (UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8))

Can it be converted into a structure or named tuple or something?

tobias.knopp · 2019-02-02 07:56:37 UTC

you have to do this manually. I have an example for reading here:

github.com

MagneticResonanceImaging/MRIReco.jl/blob/master/src/IO/ISMRMRD.jl#L59


data::Array{Complex{Float32},3}
xdoc
end


function ISMRMRD(filename::String)
headerStr = h5read(filename, "/dataset/xml")
xdoc = parse_string(headerStr[1])


header = parse_xml_header(xdoc)


d = h5read(filename, "/dataset/data")


M = length(d)


head = ISMRMRDAcquisitionHeader[]
traj = Float32[]
data = Any[]


chan = header["receiverChannels"]


for m=1:M

And here is an example for writing a compound data type:

github.com

MagneticParticleImaging/MPIFiles.jl/blob/master/src/MPIFiles.jl#L384


  return BrukerFile(filename; kargs...)
end
end


# Opens a set of MPIFiles
function MPIFile(filenames::Vector)
return map(x->MPIFile(x),filenames)
end


# Support for handling complex datatypes in HDF5 files
function writeComplexArray(file, dataset, A::AbstractArray{Complex{T},D}) where {T,D}
d_type_compound = HDF5.h5t_create(HDF5.H5T_COMPOUND,2*sizeof(T))
HDF5.h5t_insert(d_type_compound, "r", 0 , HDF5.hdf5_type_id(T))
HDF5.h5t_insert(d_type_compound, "i", sizeof(T) , HDF5.hdf5_type_id(T))


shape = collect(reverse(size(A)))
space = HDF5.h5s_create_simple(D, shape, shape)


dset_compound = HDF5.h5d_create(file, dataset, d_type_compound, space,
                                HDF5.H5P_DEFAULT,HDF5.H5P_DEFAULT,HDF5.H5P_DEFAULT)
HDF5.h5s_close(space)

Of course it would be great to have something which does this automatically.

tamasgal · 2019-02-02 10:09:56 UTC

I have these two functions to read compound structures, given that you have a struct with the corresponding layout:

function read_compound(dset::HDF5.HDF5Dataset, T::DataType)
    filetype = HDF5.datatype(dset)
    memtype_id = HDF5.h5t_get_native_type(filetype.id)
    @assert sizeof(T) == HDF5.h5t_get_size(memtype_id) "type size mismatch"
    out = Vector{T}(undef, length(dset))
    HDF5.h5d_read(dset.id, memtype_id, HDF5.H5S_ALL, HDF5.H5S_ALL, HDF5.H5P_DEFAULT, out)
    HDF5.h5t_close(memtype_id)
    out
end


function read_compound(filename::AbstractString,
                       h5loc::AbstractString,
                       T::DataType)
    fobj = HDF5.h5open(filename)
    data = read_compound(fobj[h5loc], T)
    close(fobj)
    data
end