Efficiently loading a batch of .mat files

Hello, I am new at optimizing julia code. Currently I am working on a project translating a model in MATLAB to Julia. There is a large amount of starting data for this model written in .mat files.

Right now, I am developing a struct to hold a large amount of complex data (it was originally a cell array in MATLAB). The struct does not need to be mutable.



# create tau
struct StructTau
    tau::Vector{Matrix{Float64}}
    tau1s::Matrix{Float64} 

    # Inner Constructor
    StructTau() = new(
        [Matrix{Float64}(undef, 2531, 2531) for _ in 1:10],  # Pre-allocated tau vector
        Matrix{Float64}(undef, 2531, 2531) # Pre-allocated tau1s
    )
end


function create_tau!(Tau::StructTau, tau_paths::Vector{String})

    # Load each Tau matrix from its respective file into the pre-allocated matrices
    for i in 1:10
        tau_file = matopen(tau_paths[i])
        Tau.tau[i] .= read(tau_file, "tau_export")::Matrix{Float64}
        close(tau_file)
    end

    Tau.tau1s .= Tau.tau[end]
end

tau_paths = ["Data/ModelDataset/tau/tau_ind1.mat", "Data/ModelDataset/tau/tau_ind2.mat",
            "Data/ModelDataset/tau/tau_ind3.mat", "Data/ModelDataset/tau/tau_ind4.mat",
            "Data/ModelDataset/tau/tau_ind5.mat", "Data/ModelDataset/tau/tau_ind6.mat", 
            "Data/ModelDataset/tau/tau_ind7.mat", "Data/ModelDataset/tau/tau_ind8.mat",
            "Data/ModelDataset/tau/tau_ind9.mat", "Data/ModelDataset/tau/tau_ind10.mat"]
Tau = StructTau()
create_tau!(Tau, tau_paths)

Using @time shows:
1.978473 seconds (780 allocations: 672.607 MiB, 1.78% gc time)

When I run @code_warntype there is an Any and a couple of Unions that I am not sure how to handle:

  from create_tau!(Tau::StructTau, tau_paths::Vector{String}) @ Main c:\Users\vrg2121\Dropbox (CBS)\Conor Walsh\TradeAndGrowth\Code\RWCodeJulia\RWCodeJulia\src\tau_read.jl:17
Arguments
  #self#::Core.Const(create_tau!)
  Tau::StructTau
  tau_paths::Vector{String}
Locals
  @_4::Union{Nothing, Tuple{Int64, Int64}}
  i::Int64
  tau_file::Union{MAT.MAT_HDF5.MatlabHDF5File, MAT.MAT_v4.Matlabv4File, MAT.MAT_v5.Matlabv5File}
Body::Matrix{Float64}
1 ─ %1  = (1:10)::Core.Const(1:10)
│         (@_4 = Base.iterate(%1))
│   %3  = (@_4::Core.Const((1, 1)) === nothing)::Core.Const(false)
│   %4  = Base.not_int(%3)::Core.Const(true)
└──       goto #4 if not %4
2 ┄ %6  = @_4::Tuple{Int64, Int64}
│         (i = Core.getfield(%6, 1))
│   %8  = Core.getfield(%6, 2)::Int64
│   %9  = Base.getindex(tau_paths, i)::String
│         (tau_file = Main.matopen(%9))
│   %11 = Base.getproperty(Tau, :tau)::Vector{Matrix{Float64}}
│   %12 = Base.dotview(%11, i)::Matrix{Float64}
│   %13 = Main.read(tau_file, "tau_export")::Any
│   %14 = Core.apply_type(Main.Matrix, Main.Float64)::Core.Const(Matrix{Float64})
│   %15 = Core.typeassert(%13, %14)::Matrix{Float64}
│   %16 = Base.broadcasted(Base.identity, %15)::Base.Broadcast.Broadcasted{Base.Broadcast.DefaultArrayStyle{2}, Nothing, typeof(identity), Tuple{Matrix{Float64}}}
│         Base.materialize!(%12, %16)
│         Main.close(tau_file)
│         (@_4 = Base.iterate(%1, %8))
│   %20 = (@_4 === nothing)::Bool
│   %21 = Base.not_int(%20)::Bool
└──       goto #4 if not %21
3 ─       goto #2
4 ┄ %24 = Base.dotgetproperty(Tau, :tau1s)::Matrix{Float64}
│   %25 = Base.getproperty(Tau, :tau)::Vector{Matrix{Float64}}
│   %26 = Base.lastindex(%25)::Int64
│   %27 = Base.getindex(%25, %26)::Matrix{Float64}
│   %28 = Base.broadcasted(Base.identity, %27)::Base.Broadcast.Broadcasted{Base.Broadcast.DefaultArrayStyle{2}, Nothing, typeof(identity), Tuple{Matrix{Float64}}}
│   %29 = Base.materialize!(%24, %28)::Matrix{Float64}
└──       return %29

And finally, Profile.print() is below. However, I am not very good at understanding it.

Overhead ╎ [+additional indent] Count File:Line; Function
=========================================================
    ╎2093 @VSCodeServer\src\eval.jl:34; (::VSCodeServer.var"#64#65")()
    ╎ 2093 @Base\essentials.jl:889; invokelatest(::Any)
    ╎  2093 @Base\essentials.jl:892; #invokelatest#2
    ╎   2093 @VSCodeServer\src\eval.jl:255; (::VSCodeServer.var"#66#71"{VSCodeServer.ReplRunCodeRequestParams})()
    ╎    2093 @Base\logging.jl:627; with_logger
    ╎     2093 @Base\logging.jl:515; with_logstate(f::Function, logstate::Any)
    ╎    ╎ 2093 @VSCodeServer\src\eval.jl:150; (::VSCodeServer.var"#67#72"{Bool, Bool, Bool, Module, String, Int64, Int64, String, VSCodeServer.ReplRunCodeRequestParams})()
    ╎    ╎  2093 @VSCodeServer\src\repl.jl:38; hideprompt(f::VSCodeServer.var"#68#73"{Bool, Bool, Bool, Module, String, Int64, Int64, String, VSCodeServer.ReplRunCodeRequestParams})
    ╎    ╎   2093 @VSCodeServer\src\eval.jl:179; (::VSCodeServer.var"#68#73"{Bool, Bool, Bool, Module, String, Int64, Int64, String, VSCodeServer.ReplRunCodeRequestParams})()
    ╎    ╎    2093 @VSCodeServer\src\repl.jl:274; withpath(f::VSCodeServer.var"#69#74"{Bool, Bool, Bool, Module, String, Int64, Int64, String, VSCodeServer.ReplRunCodeRequestParams}, path::String)
    ╎    ╎     2093 @VSCodeServer\src\eval.jl:181; (::VSCodeServer.var"#69#74"{Bool, Bool, Bool, Module, String, Int64, Int64, String, VSCodeServer.ReplRunCodeRequestParams})()
    ╎    ╎    ╎ 2093 @VSCodeServer\src\eval.jl:260; kwcall(::@NamedTuple{softscope::Bool}, ::typeof(VSCodeServer.inlineeval), m::Module, code::String, code_line::Int64, code_column::Int64, file::String)
    ╎    ╎    ╎  2093 @VSCodeServer\src\eval.jl:263; inlineeval(m::Module, code::String, code_line::Int64, code_column::Int64, file::String; softscope::Bool)
    ╎    ╎    ╎   2093 @Base\essentials.jl:889; invokelatest(::Any, ::Any, ::Vararg{Any})
    ╎    ╎    ╎    2093 @Base\essentials.jl:892; invokelatest(::Any, ::Any, ::Vararg{Any}; kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, @NamedTuple{}})
    ╎    ╎    ╎     2093 @Base\loading.jl:2076; include_string(mapexpr::typeof(REPL.softscope), mod::Module, code::String, filename::String)
   2╎    ╎    ╎    ╎ 2093 @Base\boot.jl:385; eval
    ╎    ╎    ╎    ╎  1    @Base\compiler\typeinfer.jl:1078; typeinf_ext_toplevel(mi::Core.MethodInstance, world::UInt64)
    ╎    ╎    ╎    ╎   1    @Base\compiler\typeinfer.jl:1082; typeinf_ext_toplevel(interp::Core.Compiler.NativeInterpreter, linfo::Core.MethodInstance)
    ╎    ╎    ╎    ╎    1    @Base\compiler\typeinfer.jl:1049; typeinf_ext(interp::Core.Compiler.NativeInterpreter, mi::Core.MethodInstance)
    ╎    ╎    ╎    ╎     1    @Base\compiler\inferencestate.jl:430; Core.Compiler.InferenceState(result::Core.Compiler.InferenceResult, cache::Symbol, interp::Core.Compiler.NativeInterpreter)
   1╎    ╎    ╎    ╎    ╎ 1    @Base\compiler\utilities.jl:143; retrieve_code_info
    ╎    ╎    ╎    ╎  2    @RWCodeJulia\src\tau_read.jl:21; create_tau!(Tau::StructTau, tau_paths::Vector{String})
    ╎    ╎    ╎    ╎   2    @MAT\src\MAT.jl:97; matopen
    ╎    ╎    ╎    ╎    2    @MAT\src\MAT.jl:97; #matopen#2
    ╎    ╎    ╎    ╎     2    @MAT\src\MAT.jl:87; matopen
    ╎    ╎    ╎    ╎    ╎ 2    @MAT\src\MAT.jl:88; matopen(fname::String, mode::String; compress::Bool)
    ╎    ╎    ╎    ╎    ╎  1    @MAT\src\MAT.jl:41; matopen(filename::String, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool)
    ╎    ╎    ╎    ╎    ╎   1    @Base\stat.jl:461; filesize
   1╎    ╎    ╎    ╎    ╎    1    @Base\stat.jl:150; stat(path::String)
    ╎    ╎    ╎    ╎    ╎  1    @MAT\src\MAT.jl:48; matopen(filename::String, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool)
    ╎    ╎    ╎    ╎    ╎   1    @Base\iostream.jl:355; open
    ╎    ╎    ╎    ╎    ╎    1    @Base\iostream.jl:356; open(fname::String, mode::String; lock::Bool)
    ╎    ╎    ╎    ╎    ╎     1    @Base\iostream.jl:275; open
   1╎    ╎    ╎    ╎    ╎    ╎ 1    @Base\iostream.jl:293; open(fname::String; lock::Bool, read::Bool, write::Nothing, create::Nothing, truncate::Nothing, append::Nothing)
    ╎    ╎    ╎    ╎  2085 @RWCodeJulia\src\tau_read.jl:22; create_tau!(Tau::StructTau, tau_paths::Vector{String})
    ╎    ╎    ╎    ╎   37   @Base\broadcast.jl:911; materialize!
    ╎    ╎    ╎    ╎    37   @Base\broadcast.jl:914; materialize!
    ╎    ╎    ╎    ╎     37   @Base\broadcast.jl:956; copyto!
    ╎    ╎    ╎    ╎    ╎ 37   @Base\broadcast.jl:997; copyto!
    ╎    ╎    ╎    ╎    ╎  37   @Base\array.jl:388; copyto!
    ╎    ╎    ╎    ╎    ╎   37   @Base\array.jl:368; copyto!
    ╎    ╎    ╎    ╎    ╎    37   @Base\array.jl:376; _copyto_impl!(dest::Matrix{Float64}, doffs::Int64, src::Matrix{Float64}, soffs::Int64, n::Int64)
    ╎    ╎    ╎    ╎    ╎     37   @Base\array.jl:337; unsafe_copyto!
  37╎    ╎    ╎    ╎    ╎    ╎ 37   @Base\cmem.jl:26; memmove
    ╎    ╎    ╎    ╎   2    @MAT\src\MAT_v5.jl:420; read(matfile::MAT.MAT_v5.Matlabv5File, varname::String)
    ╎    ╎    ╎    ╎    1    @MAT\src\MAT_v5.jl:395; getvarnames(matfile::MAT.MAT_v5.Matlabv5File)
    ╎    ╎    ╎    ╎     1    @CodecZlib\src\decompression.jl:82; TranscodingStream
    ╎    ╎    ╎    ╎    ╎ 1    @CodecZlib\src\decompression.jl:84; #ZlibDecompressorStream#10
    ╎    ╎    ╎    ╎    ╎  1    @TranscodingStreams\src\stream.jl:110; TranscodingStream
    ╎    ╎    ╎    ╎    ╎   1    @TranscodingStreams\src\stream.jl:126; TranscodingStreams.TranscodingStream(codec::CodecZlib.ZlibDecompressor, stream::IOStream; bufsize::Int64, stop_on_end::Bool, sharedbuf::Bool)
    ╎    ╎    ╎    ╎    ╎    1    @TranscodingStreams\src\stream.jl:37; TranscodingStream
    ╎    ╎    ╎    ╎    ╎     1    @TranscodingStreams\src\stream.jl:39; #TranscodingStream#4
    ╎    ╎    ╎    ╎    ╎    ╎ 1    @TranscodingStreams\src\stream.jl:31; TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOStream}(codec::CodecZlib.ZlibDecompressor, stream::IOStream, state::TranscodingStreams.State, initialized::Bool)
    ╎    ╎    ╎    ╎    ╎    ╎  1    @CodecZlib\src\decompression.jl:129; initialize
    ╎    ╎    ╎    ╎    ╎    ╎   1    @CodecZlib\src\libz.jl:81; inflate_init!(zstream::CodecZlib.ZStream, windowbits::Int64)
   1╎    ╎    ╎    ╎    ╎    ╎    1    @Base\c.jl:196; cconvert(::Type{Cstring}, s::String)
    ╎    ╎    ╎    ╎    1    @MAT\src\MAT_v5.jl:396; getvarnames(matfile::MAT.MAT_v5.Matlabv5File)
    ╎    ╎    ╎    ╎     1    @MAT\src\MAT_v5.jl:110; read_header
    ╎    ╎    ╎    ╎    ╎ 1    @MAT\src\MAT_v5.jl:83; read_bswap
    ╎    ╎    ╎    ╎    ╎  1    @Base\io.jl:779; read
    ╎    ╎    ╎    ╎    ╎   1    @Base\io.jl:775; read!
    ╎    ╎    ╎    ╎    ╎    1    @Base\io.jl:773; unsafe_read(s::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOStream}, p::Base.RefValue{UInt32}, n::Int64)
    ╎    ╎    ╎    ╎    ╎     1    @Base\io.jl:774; unsafe_read
    ╎    ╎    ╎    ╎    ╎    ╎ 1    @TranscodingStreams\src\stream.jl:366; unsafe_read(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOStream}, output::Ptr{UInt8}, nbytes::UInt64)
    ╎    ╎    ╎    ╎    ╎    ╎  1    @TranscodingStreams\src\stream.jl:198; eof(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOStream})
    ╎    ╎    ╎    ╎    ╎    ╎   1    @TranscodingStreams\src\stream.jl:579; fillbuffer
    ╎    ╎    ╎    ╎    ╎    ╎    1    @TranscodingStreams\src\stream.jl:593; fillbuffer(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOStream}; eager::Bool)
    ╎    ╎    ╎    ╎    ╎    ╎     1    @TranscodingStreams\src\stream.jl:653; callprocess(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOStream}, inbuf::TranscodingStreams.Buffer, outbuf::TranscodingStreams.Buffer)
    ╎    ╎    ╎    ╎    ╎    ╎    ╎ 1    @CodecZlib\src\decompression.jl:166; process(codec::CodecZlib.ZlibDecompressor, input::TranscodingStreams.Memory, output::TranscodingStreams.Memory, error::TranscodingStreams.Error)
   1╎    ╎    ╎    ╎    ╎    ╎    ╎  1    @CodecZlib\src\libz.jl:93; inflate!
    ╎    ╎    ╎    ╎   2046 @MAT\src\MAT_v5.jl:425; read(matfile::MAT.MAT_v5.Matlabv5File, varname::String)
    ╎    ╎    ╎    ╎    2046 @MAT\src\MAT_v5.jl:318; read_matrix(f::IOStream, swap_bytes::Bool)
   6╎    ╎    ╎    ╎     6    @Base\boot.jl:477; Array
    ╎    ╎    ╎    ╎     58   @Base\io.jl:786; read!
    ╎    ╎    ╎    ╎    ╎ 58   @Base\io.jl:774; unsafe_read
  57╎    ╎    ╎    ╎    ╎  58   @Base\iostream.jl:43; unsafe_read(s::IOStream, p::Ptr{UInt8}, nb::UInt64)
    ╎    ╎    ╎    ╎     1    @MAT\src\MAT_v5.jl:316; read_matrix(f::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}, swap_bytes::Bool)
    ╎    ╎    ╎    ╎    ╎ 1    @MAT\src\MAT_v5.jl:110; read_header
    ╎    ╎    ╎    ╎    ╎  1    @MAT\src\MAT_v5.jl:83; read_bswap
    ╎    ╎    ╎    ╎    ╎   1    @Base\io.jl:779; read
    ╎    ╎    ╎    ╎    ╎    1    @Base\io.jl:775; read!
    ╎    ╎    ╎    ╎    ╎     1    @Base\io.jl:773; unsafe_read(s::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}, p::Base.RefValue{UInt32}, n::Int64)
    ╎    ╎    ╎    ╎    ╎    ╎ 1    @Base\io.jl:774; unsafe_read
    ╎    ╎    ╎    ╎    ╎    ╎  1    @TranscodingStreams\src\stream.jl:366; unsafe_read(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}, output::Ptr{UInt8}, nbytes::UInt64)
    ╎    ╎    ╎    ╎    ╎    ╎   1    @TranscodingStreams\src\stream.jl:198; eof
    ╎    ╎    ╎    ╎    ╎    ╎    1    @TranscodingStreams\src\stream.jl:579; fillbuffer
    ╎    ╎    ╎    ╎    ╎    ╎     1    @TranscodingStreams\src\stream.jl:593; fillbuffer(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}; eager::Bool)
    ╎    ╎    ╎    ╎    ╎    ╎    ╎ 1    @TranscodingStreams\src\stream.jl:653; callprocess(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}, inbuf::TranscodingStreams.Buffer, outbuf::TranscodingStreams.Buffer)
    ╎    ╎    ╎    ╎    ╎    ╎    ╎  1    @CodecZlib\src\decompression.jl:166; process(codec::CodecZlib.ZlibDecompressor, input::TranscodingStreams.Memory, output::TranscodingStreams.Memory, error::TranscodingStreams.Error)
   1╎    ╎    ╎    ╎    ╎    ╎    ╎   1    @CodecZlib\src\libz.jl:93; inflate!
    ╎    ╎    ╎    ╎     1981 @MAT\src\MAT_v5.jl:362; read_matrix(f::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}, swap_bytes::Bool)
    ╎    ╎    ╎    ╎    ╎ 1981 @MAT\src\MAT_v5.jl:153; read_data(f::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}, swap_bytes::Bool, ::Type{Float64}, dimensions::Vector{Int32})
    ╎    ╎    ╎    ╎    ╎  1981 @MAT\src\MAT_v5.jl:86; read_bswap(f::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}, swap_bytes::Bool, ::Type{Float64}, dim::Tuple{Int64, Int64})
    ╎    ╎    ╎    ╎    ╎   16   @Base\boot.jl:494; Array
    ╎    ╎    ╎    ╎    ╎    16   @Base\boot.jl:487; Array
  16╎    ╎    ╎    ╎    ╎     16   @Base\boot.jl:479; Array
    ╎    ╎    ╎    ╎    ╎   1965 @Base\io.jl:792; read!
    ╎    ╎    ╎    ╎    ╎    1965 @Base\io.jl:774; unsafe_read
    ╎    ╎    ╎    ╎    ╎     1859 @TranscodingStreams\src\stream.jl:366; unsafe_read(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}, output::Ptr{UInt8}, nbytes::UInt64)
    ╎    ╎    ╎    ╎    ╎    ╎ 1859 @TranscodingStreams\src\stream.jl:198; eof
    ╎    ╎    ╎    ╎    ╎    ╎  1859 @TranscodingStreams\src\stream.jl:579; fillbuffer
    ╎    ╎    ╎    ╎    ╎    ╎   5    @TranscodingStreams\src\stream.jl:591; fillbuffer(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}; eager::Bool)
    ╎    ╎    ╎    ╎    ╎    ╎    5    @TranscodingStreams\src\buffer.jl:128; makemargin!
    ╎    ╎    ╎    ╎    ╎    ╎     5    @TranscodingStreams\src\buffer.jl:147; makemargin!(buf::TranscodingStreams.Buffer, minsize::Int64; eager::Bool)
    ╎    ╎    ╎    ╎    ╎    ╎    ╎ 5    @Base\array.jl:368; copyto!
    ╎    ╎    ╎    ╎    ╎    ╎    ╎  5    @Base\array.jl:376; _copyto_impl!(dest::Vector{UInt8}, doffs::Int64, src::Vector{UInt8}, soffs::Int64, n::Int64)
    ╎    ╎    ╎    ╎    ╎    ╎    ╎   5    @Base\array.jl:337; unsafe_copyto!
   5╎    ╎    ╎    ╎    ╎    ╎    ╎    5    @Base\cmem.jl:26; memmove
    ╎    ╎    ╎    ╎    ╎    ╎   14   @TranscodingStreams\src\stream.jl:592; fillbuffer(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}; eager::Bool)
    ╎    ╎    ╎    ╎    ╎    ╎    14   @TranscodingStreams\src\stream.jl:693; readdata!(input::IOBuffer, output::TranscodingStreams.Buffer)
    ╎    ╎    ╎    ╎    ╎    ╎     14   @Base\io.jl:774; unsafe_read
    ╎    ╎    ╎    ╎    ╎    ╎    ╎ 14   @Base\iobuffer.jl:172; unsafe_read
    ╎    ╎    ╎    ╎    ╎    ╎    ╎  14   @Base\array.jl:288; unsafe_copyto!
  14╎    ╎    ╎    ╎    ╎    ╎    ╎   14   @Base\cmem.jl:26; memmove
   1╎    ╎    ╎    ╎    ╎    ╎   1840 @TranscodingStreams\src\stream.jl:593; fillbuffer(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}; eager::Bool)
    ╎    ╎    ╎    ╎    ╎    ╎    1836 @TranscodingStreams\src\stream.jl:653; callprocess(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}, inbuf::TranscodingStreams.Buffer, outbuf::TranscodingStreams.Buffer)
    ╎    ╎    ╎    ╎    ╎    ╎     1836 @CodecZlib\src\decompression.jl:166; process(codec::CodecZlib.ZlibDecompressor, input::TranscodingStreams.Memory, output::TranscodingStreams.Memory, error::TranscodingStreams.Error)
1836╎    ╎    ╎    ╎    ╎    ╎    ╎ 1836 @CodecZlib\src\libz.jl:93; inflate!
    ╎    ╎    ╎    ╎    ╎    ╎    3    @TranscodingStreams\src\stream.jl:654; callprocess(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}, inbuf::TranscodingStreams.Buffer, outbuf::TranscodingStreams.Buffer)
    ╎    ╎    ╎    ╎    ╎    ╎     3    @Base\logging.jl:389; macro expansion
    ╎    ╎    ╎    ╎    ╎    ╎    ╎ 3    @Base\logging.jl:328; current_logger_for_env(std_level::Base.CoreLogging.LogLevel, group::Symbol, _module::Module)
    ╎    ╎    ╎    ╎    ╎    ╎    ╎  3    @Base\logging.jl:544; env_override_minlevel(group::Symbol, _module::Module)
    ╎    ╎    ╎    ╎    ╎    ╎    ╎   3    @Base\env.jl:157; get
    ╎    ╎    ╎    ╎    ╎    ╎    ╎    1    @Base\env.jl:29; access_env(onError::Returns{String}, str::String)
    ╎    ╎    ╎    ╎    ╎    ╎    ╎     1    @Base\env.jl:14; memoized_env_lookup(str::String)
    ╎    ╎    ╎    ╎    ╎    ╎    ╎    ╎ 1    @Base\lock.jl:269; macro expansion
    ╎    ╎    ╎    ╎    ╎    ╎    ╎    ╎  1    @Base\lock.jl:177; unlock
    ╎    ╎    ╎    ╎    ╎    ╎    ╎    ╎   1    @Base\lock.jl:182; (::Base.var"#_unlock#648")(rl::ReentrantLock)
   1╎    ╎    ╎    ╎    ╎    ╎    ╎    ╎    1    @Base\Base.jl:70; swapproperty!
    ╎    ╎    ╎    ╎    ╎    ╎    ╎    2    @Base\env.jl:30; access_env(onError::Returns{String}, str::String)
   2╎    ╎    ╎    ╎    ╎    ╎    ╎     2    @Base\env.jl:24; _getenvlen
    ╎    ╎    ╎    ╎    ╎     106  @TranscodingStreams\src\stream.jl:368; unsafe_read(stream::TranscodingStreams.TranscodingStream{CodecZlib.ZlibDecompressor, IOBuffer}, output::Ptr{UInt8}, nbytes::UInt64)
    ╎    ╎    ╎    ╎    ╎    ╎ 106  @TranscodingStreams\src\buffer.jl:213; copydata!
    ╎    ╎    ╎    ╎    ╎    ╎  106  @Base\array.jl:288; unsafe_copyto!
 106╎    ╎    ╎    ╎    ╎    ╎   106  @Base\cmem.jl:26; memmove
    ╎    ╎    ╎    ╎  1    @RWCodeJulia\src\tau_read.jl:23; create_tau!(Tau::StructTau, tau_paths::Vector{String})
    ╎    ╎    ╎    ╎   1    @MAT\src\MAT_v5.jl:435; close
   1╎    ╎    ╎    ╎    1    @Base\iostream.jl:43; close(s::IOStream)
    ╎    ╎    ╎    ╎  2    @RWCodeJulia\src\tau_read.jl:26; create_tau!(Tau::StructTau, tau_paths::Vector{String})
    ╎    ╎    ╎    ╎   2    @Base\broadcast.jl:911; materialize!
    ╎    ╎    ╎    ╎    2    @Base\broadcast.jl:914; materialize!
    ╎    ╎    ╎    ╎     2    @Base\broadcast.jl:956; copyto!
    ╎    ╎    ╎    ╎    ╎ 2    @Base\broadcast.jl:997; copyto!
    ╎    ╎    ╎    ╎    ╎  2    @Base\array.jl:388; copyto!
    ╎    ╎    ╎    ╎    ╎   2    @Base\array.jl:368; copyto!
    ╎    ╎    ╎    ╎    ╎    2    @Base\array.jl:376; _copyto_impl!(dest::Matrix{Float64}, doffs::Int64, src::Matrix{Float64}, soffs::Int64, n::Int64)
    ╎    ╎    ╎    ╎    ╎     2    @Base\array.jl:337; unsafe_copyto!
   2╎    ╎    ╎    ╎    ╎    ╎ 2    @Base\cmem.jl:26; memmove

Thank you for your help!

This Any:

Main.read(tau_file, "tau_export")::Any

is unavoidable since the read method needs to be flexible. However, you already confine that type instability with the type assertion

read(tau_file, "tau_export")::Matrix{Float64}

which is as tight as you can do this when reading from a .mat file.

The union @_4::Union{Nothing, Tuple{Int64, Int64}} comes from the iteration interface and is a small union which is handled efficiently with union splitting. This is nothing to worry about.

I find it easier to understand profile output with some graphical interface like ProfileView or Profile.print(format=:flat) but in this case it appears like all time is spent here:

1836 @CodecZlib\src\libz.jl:93; inflate!

This means that the .mat file is compressed and that nearly all time reading it is spent on doing decompression.

1 Like

Thanks for the explanation of the profiling. I have been considering a more efficient data format like HDF5. But I do not know the format well enough to know if it will fix the decompression issue.