Write structure to HDF5 file and read a compound type from HDF5 into Julia structure

sairus7 · February 1, 2019, 4:04pm

How to write a structure in HDF5 file?

using HDF5
mutable struct Foo
    a::Int
    b::Float64
end
vect = [1,2,3,4,5]
foo = Foo(1, 2.0)
foovect = [P(i, i/2) for i=1:5]
path = "/testfile.h5"
h5write(path, "/vect", vect)
h5write(path, "/foo", foo) 
# > MethodError: no method matching write(::HDF5File, ::String, ::Foo)
h5write(path, "/foovec", foovect)
# > MethodError: no method matching write(::HDF5File, ::String, ::Array{P,1})

And how to read a compound type?
If I write some compound type into a hdf5 with c++ (Hello World! into a byte fields named as field0 … field11), then try to read it in Julia, it reads an array of HDF5Compound type:

HDF5.HDF5Compound{12}
data -> (0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21), 
membername -> ("0\xfd\xf6\x11field0", "0\xfd\xf6\x11field1", "0\xfd\xf6\x11field2", "0\xfd\xf6\x11field3", "0\xfd\xf6\x11field4", "0\xfd\xf6\x11field5", "0\xfd\xf6\x11field6", "0\xfd\xf6\x11field7", "0\xfd\xf6\x11field8", "0\xfd\xf6\x11field9", "0\xfd\xf6\x11field10", "0\xfd\xf6\x11field11"), 
membertype -> (UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8, UInt8))

Can it be converted into a structure or named tuple or something?

tobias.knopp · February 2, 2019, 7:56am

you have to do this manually. I have an example for reading here:

https://github.com/MagneticResonanceImaging/MRIReco.jl/blob/master/src/IO/ISMRMRD.jl#L59

And here is an example for writing a compound data type:

github.com

MagneticParticleImaging/MPIFiles.jl/blob/master/src/MPIFiles.jl#L384


      
          module MPIFiles
          
          
using UUIDs
          using Graphics: @mustimplement
          
          
using Reexport
          @reexport using SparsityOperators
          @reexport using AxisArrays
          const axes = Base.axes
          @reexport using Interpolations
          @reexport using HDF5
          @reexport using Dates
          @reexport using DelimitedFiles
          @reexport using Images
          @reexport using LinearAlgebra
          @reexport using Random
          @reexport using Mmap
          @reexport using Statistics
          @reexport using Unitful
          using CodecZlib
          using Tar
          using Pkg.PlatformEngines
          using Pkg.GitTools
          using Pkg.Artifacts
          using Unitful
          using InteractiveUtils
          using UnitfulAngles
          using Inflate, SHA
          using StableRNGs
          using REPL: fielddoc
          using DocStringExtensions
          
          
if VERSION < v"1.1"
            isnothing(x) = x == nothing
          end
          
          
### global import list ###
          
          
import Base: convert, get, getindex, haskey, iterate, length, ndims, range, read, show, time, write, close
          import FileIO: save
          import HDF5: h5read
          import Interpolations: interpolate
          
          
### export list ###
          
          
export MPIFile
          
          
# general parameters
          export version, uuid
          
          
# study parameters
          export studyName, studyNumber, studyUuid, studyDescription, studyTime
          
          
# experiment parameters
          export experimentName, experimentNumber, experimentUuid, experimentDescription, experimentSubject,
                experimentIsSimulation, experimentIsCalibration,
                experimentHasMeasurement, experimentHasReconstruction
          
          
# tracer parameters
          export tracerName, tracerBatch, tracerVolume, tracerConcentration,
                 tracerSolute, tracerInjectionTime, tracerVendor
          
          
# scanner parameters
          export scannerFacility, scannerOperator, scannerManufacturer, scannerName,
                 scannerTopology
          
          
# acquisition parameters
          export acqStartTime, acqNumFrames, acqNumAverages,
                 acqGradient, acqOffsetField, acqNumPeriodsPerFrame, acqSize
          
          
# drive-field parameters
          export dfNumChannels, dfStrength, dfPhase, dfBaseFrequency, dfCustomWaveform,
                 dfDivider, dfWaveform, dfCycle
          
          
# receiver parameters
          export rxNumChannels, rxBandwidth, rxNumSamplingPoints,
                 rxTransferFunction, rxTransferFunctionFileName, rxHasTransferFunction, rxUnit,
                 rxDataConversionFactor, rxInductionFactor
          
          
# measurements
          export measData, measDataTDPeriods, measIsFourierTransformed, measIsTFCorrected,
                 measIsTransferFunctionCorrected,
                 measIsBGCorrected, measIsBackgroundCorrected, measIsFastFrameAxis,
                 measIsFramePermutation, measIsFrequencySelection,
                 measIsBGFrame, measIsBackgroundFrame, measIsSpectralLeakageCorrected, measFramePermutation,
                 measFrequencySelection, measIsSparsityTransformed, measIsCalibProcessed
          
          
# calibrations
          export calibSNR, calibSnr, calibFov, calibFieldOfView, calibFovCenter,
                 calibFieldOfViewCenter, calibSize, calibOrder, calibPositions,
                 calibOffsetField, calibDeltaSampleSize,
                 calibMethod, calibIsMeanderingGrid
          
          
# reconstruction results
          export recoData, recoFov, recoFieldOfView, recoFovCenter, recoFieldOfViewCenter,
                 recoSize, recoOrder, recoPositions
          
          
# additional functions that should be implemented by an MPIFile
          export filepath, systemMatrixWithBG, systemMatrix
          
          
export selectedChannels
          ### Interface of an MPIFile ###
          
          
abstract type MPIFile end
          
          
# general parameters
          @mustimplement version(f::MPIFile)
          @mustimplement uuid(f::MPIFile)
          @mustimplement time(f::MPIFile)
          
          
# study parameters
          @mustimplement studyName(f::MPIFile)
          @mustimplement studyNumber(f::MPIFile)
          @mustimplement studyUuid(f::MPIFile)
          @mustimplement studyDescription(f::MPIFile)
          @mustimplement studyTime(f::MPIFile)
          
          
# experiment parameters
          @mustimplement experimentName(f::MPIFile)
          @mustimplement experimentNumber(f::MPIFile)
          @mustimplement experimentUuid(f::MPIFile)
          @mustimplement experimentDescription(f::MPIFile)
          @mustimplement experimentSubject(f::MPIFile)
          @mustimplement experimentIsSimulation(f::MPIFile)
          @mustimplement experimentIsCalibration(f::MPIFile)
          @mustimplement experimentHasReconstruction(f::MPIFile)
          @mustimplement experimentHasMeasurement(f::MPIFile)
          
          
# tracer parameters
          @mustimplement tracerName(f::MPIFile)
          @mustimplement tracerBatch(f::MPIFile)
          @mustimplement tracerVolume(f::MPIFile)
          @mustimplement tracerConcentration(f::MPIFile)
          @mustimplement tracerSolute(f::MPIFile)
          @mustimplement tracerInjectionTime(f::MPIFile)
          
          
# scanner parameters
          @mustimplement scannerBoreSize(f::MPIFile)
          @mustimplement scannerFacility(f::MPIFile)
          @mustimplement scannerOperator(f::MPIFile)
          @mustimplement scannerManufacturer(f::MPIFile)
          @mustimplement scannerName(f::MPIFile)
          @mustimplement scannerTopology(f::MPIFile)
          
          
# acquisition parameters
          @mustimplement acqStartTime(f::MPIFile)
          @mustimplement acqNumAverages(f::MPIFile)
          @mustimplement acqNumPeriodsPerFrame(f::MPIFile)
          @mustimplement acqNumFrames(f::MPIFile)
          @mustimplement acqGradient(f::MPIFile)
          @mustimplement acqOffsetField(f::MPIFile)
          
          
# drive-field parameters
          @mustimplement dfNumChannels(f::MPIFile)
          @mustimplement dfStrength(f::MPIFile)
          @mustimplement dfPhase(f::MPIFile)
          @mustimplement dfBaseFrequency(f::MPIFile)
          @mustimplement dfCustomWaveform(f::MPIFile)
          @mustimplement dfDivider(f::MPIFile)
          @mustimplement dfWaveform(f::MPIFile)
          @mustimplement dfCycle(f::MPIFile)
          
          
# receiver properties
          @mustimplement rxNumChannels(f::MPIFile)
          @mustimplement rxBandwidth(f::MPIFile)
          @mustimplement rxNumSamplingPoints(f::MPIFile)
          @mustimplement rxTransferFunction(f::MPIFile)
          @mustimplement rxTransferFunctionFileName(f::MPIFile)
          @mustimplement rxHasTransferFunction(f::MPIFile)
          @mustimplement rxInductionFactor(f::MPIFile)
          @mustimplement rxUnit(f::MPIFile)
          @mustimplement rxDataConversionFactor(f::MPIFile)
          
          
# measurements
          @mustimplement measData(f::MPIFile)
          @mustimplement measDataTD(f::MPIFile)
          @mustimplement measDataTDPeriods(f::MPIFile, periods)
          @mustimplement measFramePermutation(f::MPIFile)
          @mustimplement measFrequencySelection(f::MPIFile)
          @mustimplement measIsBGCorrected(f::MPIFile)
          @mustimplement measIsBGFrame(f::MPIFile)
          @mustimplement measIsFastFrameAxis(f::MPIFile)
          @mustimplement measIsFourierTransformed(f::MPIFile)
          @mustimplement measIsFramePermutation(f::MPIFile)
          @mustimplement measIsFrequencySelection(f::MPIFile)
          @mustimplement measIsSparsityTransformed(f::MPIFile)
          @mustimplement measIsSpectralLeakageCorrected(f::MPIFile)
          @mustimplement measIsTFCorrected(f::MPIFile)
          @mustimplement measSparsityTransformation(f::MPIFile)
          @mustimplement measSubsamplingIndices(f::MPIFile)
          @mustimplement measIsCalibProcessed(b::MPIFile)
          
          
# calibrations
          @mustimplement calibSNR(f::MPIFile)
          @mustimplement calibFov(f::MPIFile)
          @mustimplement calibFovCenter(f::MPIFile)
          @mustimplement calibSize(f::MPIFile)
          @mustimplement calibOrder(f::MPIFile)
          @mustimplement calibPositions(f::MPIFile)
          @mustimplement calibOffsetField(f::MPIFile)
          @mustimplement calibDeltaSampleSize(f::MPIFile)
          @mustimplement calibMethod(f::MPIFile)
          @mustimplement calibIsMeanderingGrid(f::MPIFile)
          
          
# reconstruction results
          @mustimplement recoData(f::MPIFile)
          @mustimplement recoFov(f::MPIFile)
          @mustimplement recoFovCenter(f::MPIFile)
          @mustimplement recoSize(f::MPIFile)
          @mustimplement recoOrder(f::MPIFile)
          @mustimplement recoPositions(f::MPIFile)
          @mustimplement recoIsOverscanRegion(f::MPIFile)
          
          
# additional functions that should be implemented by an MPIFile
          @mustimplement filepath(f::MPIFile)
          
          

          
include("Derived.jl")
          include("Custom.jl")
          include("Utils.jl")
          include("FramePermutation.jl")
          
          
### Concrete implementations ###
          include("MDF.jl")
          include("MDFInMemory.jl")
          include("MDFCommon.jl")
          include("Brukerfile.jl")
          include("IMT.jl")
          
          
# This dispatches on the file extension and automatically
          # generates the correct type
          function MPIFile(filename::AbstractString; kargs...)
            filenamebase, ext = splitext(filename)
            if ext == ".mdf" || ext == ".hdf" || ext == ".h5"
              file = h5open(filename,"r")
              if haskey(file, "/version")
                return MDFFile(filename, file) # MDFFile currently has no kargs
              else
                return IMTFile(filename, file; kargs...)
              end
            else
              if isfile(joinpath(filename,"mdf"))
                filenameMDF = readline(joinpath(filename,"mdf"))
                return MDFFile(filenameMDF)
              else
                return BrukerFile(filename; kargs...)
              end
            end
          end
          
          
function show(io::IO, f::MPIFile)
            print(io,supertype(typeof(f)))
            print(io,"\n\tStudy: ")
            show(io, studyName(f))
            print(io,", ")
            show(io,studyTime(f))
            print(io,"\n\tExperiment: ")
            show(io,experimentName(f))
            print(io,", ")
            show(io,acqStartTime(f))
            print(io,"\n")
          end
          
          
# Opens a set of MPIFiles
          function MPIFile(filenames::Vector)
            return map(x->MPIFile(x),filenames)
          end
          
          
# For the do block
          function MPIFile(h::Function, args...; kargs...)
            f = MPIFile(args...; kargs...)
            try
                h(f)
            finally
                close(f)
            end
          end
          
          
Base.length(f::MPIFile) = 1
          Base.close(f::MPIFile) = nothing
          
          
include("TransferFunction.jl")
          include("MultiMPIFile.jl")
          include("Measurements.jl")
          include("SystemMatrix.jl")
          include("FrequencyFilter.jl")
          include("Conversion.jl")
          include("RecoData.jl")
          include("DatasetStore/DatasetStore.jl")
          include("MixingFactors.jl")
          include("Positions/Positions.jl")
          include("MagneticFieldMeasurement.jl")
          
          
end # module

This file has been truncated. show original

Of course it would be great to have something which does this automatically.

tamasgal · February 2, 2019, 10:09am

I have these two functions to read compound structures, given that you have a struct with the corresponding layout:

function read_compound(dset::HDF5.HDF5Dataset, T::DataType)
    filetype = HDF5.datatype(dset)
    memtype_id = HDF5.h5t_get_native_type(filetype.id)
    @assert sizeof(T) == HDF5.h5t_get_size(memtype_id) "type size mismatch"
    out = Vector{T}(undef, length(dset))
    HDF5.h5d_read(dset.id, memtype_id, HDF5.H5S_ALL, HDF5.H5S_ALL, HDF5.H5P_DEFAULT, out)
    HDF5.h5t_close(memtype_id)
    out
end


function read_compound(filename::AbstractString,
                       h5loc::AbstractString,
                       T::DataType)
    fobj = HDF5.h5open(filename)
    data = read_compound(fobj[h5loc], T)
    close(fobj)
    data
end

jacob-roth · June 18, 2019, 1:58am

How can I save an array of arrays in HDF5? In particular, I’m trying to save an array of Sets, but HDF5 can’t save a Set, so I tried converting to an array, but I still cant save an array of arrays. I’m trying HDF5 over JLD2 because of what I think are similar mapping / serialization issues referenced here…