Writing MatlabOpaque objects to MAT-files (#208)

* Rearrange methods for readability

* Add some subsys methods to save objects

* Full subsystem code, some updates to writers to support writing subsystem

* Some fixes to writing filewrapper metadata

* Fix dimensions of data in filewrapper

* Write nested objects in subsystem

* Improve MAT_subsys readability

* Fix: Write empty dicts as 0x0 structs in HDF5
* subsystem shared metadata fields use these 0x0 structs

* Update empty struct handling; Add tests

* Add test for empty struct 1x1

Author: foreverallama

src/MAT.jl

@@ -204,31 +204,43 @@ Write a dictionary containing variable names as keys and values as values
 to a Matlab file, opening and closing it automatically.
 """
 function matwrite(filename::AbstractString, dict::AbstractDict{S, T}; compress::Bool = false, version::String ="v7.3") where {S, T}
-    if version == "v4"
-        file = open(filename, "w")
-        m = MAT_v4.Matlabv4File(file, false)
-        _write_dict(m, dict)
-    elseif version == "v7.3"
-        file = matopen(filename, "w"; compress = compress)
-        _write_dict(file, dict)
-    else
-        error("writing for \"$(version)\" is not supported")
+    file = nothing
+    try
+        if version == "v4"
+            file = open(filename, "w")
+            file = MAT_v4.Matlabv4File(file, false)
+            _write_dict(file, dict)
+        elseif version == "v7.3"
+            file = matopen(filename, "w"; compress = compress)
+            _write_dict(file, dict)
+        else
+            error("writing for \"$(version)\" is not supported")
+        end
+    finally
+        if file !== nothing
+            close(file)
+        end
     end
 end
 
 function _write_dict(fileio, dict::AbstractDict)
-    try
-        for (k, v) in dict
-            local kstring
-            try
-                kstring = ascii(convert(String, k))
-            catch x
-                error("matwrite requires a Dict with ASCII keys")
-            end
-            write(fileio, kstring, v)
+
+    for (k, v) in dict
+        local kstring
+        try
+            kstring = ascii(convert(String, k))
+        catch x
+            error("matwrite requires a Dict with ASCII keys")
+        end
+        write(fileio, kstring, v)
+    end
+
+    if hasproperty(fileio, :subsystem) && fileio.subsystem !== nothing
+        # will always be nothing for MATv4 so we can ignore that case
+        subsys_data = MAT_subsys.set_subsystem_data!(fileio.subsystem)
+        if subsys_data !== nothing
+            MAT_HDF5.write_subsys(fileio, subsys_data)
         end
-    finally
-        close(fileio)
     end
 end
 

src/MAT_HDF5.jl

@@ -36,7 +36,7 @@ import HDF5: Reference
 import Dates
 import Tables
 import PooledArrays: PooledArray
-import ..MAT_types: MatlabStructArray, StructArrayField, convert_struct_array, MatlabClassObject, MatlabOpaque, MatlabTable
+import ..MAT_types: MatlabStructArray, StructArrayField, convert_struct_array, MatlabClassObject, MatlabOpaque, MatlabTable, EmptyStruct
 
 const HDF5Parent = Union{HDF5.File, HDF5.Group}
 const HDF5BitsOrBool = Union{HDF5.BitsType,Bool}
@@ -130,10 +130,12 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo
         close(g)
     end
     subsys_refs = "#subsystem#"
-    if haskey(fid.plain, subsys_refs)
+    if rd && haskey(fid.plain, subsys_refs)
         fid.subsystem.table_type = table
         subsys_data = m_read(fid.plain[subsys_refs], fid.subsystem)
         MAT_subsys.load_subsys!(fid.subsystem, subsys_data, endian_indicator)
+    elseif wr
+        MAT_subsys.init_save!(fid.subsystem)
     end
     fid
 end
@@ -464,7 +466,7 @@ function _normalize_arr(x)
 end
 
 # Write a scalar or array
-function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::Union{T, Complex{T}, AbstractArray{T}, AbstractArray{Complex{T}}}) where {T<:HDF5BitsOrBool}
+function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::Union{T, Complex{T}, AbstractArray{T}, AbstractArray{Complex{T}}}, ) where {T<:HDF5BitsOrBool}
     data = _normalize_arr(data)
     if isempty(data)
         m_writeempty(parent, name, data)
@@ -541,14 +543,19 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, c::Abs
 end
 
 # Write cell arrays
-function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::AbstractArray{T}) where T
+function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::AbstractArray{T}, object_decode::UInt32=UInt32(0)) where T
     data = _normalize_arr(data)
     refs = _write_references!(mfile, parent, data)
     # Write the references as the chosen variable
     cset, ctype = create_dataset(parent, name, refs)
     try
         write_dataset(cset, ctype, refs)
-        write_attribute(cset, name_type_attr_matlab, "cell")
+        if object_decode == UInt32(3)
+            write_attribute(cset, object_decode_attr_matlab, object_decode)
+            write_attribute(cset, name_type_attr_matlab, "FileWrapper__")
+        else
+            write_attribute(cset, name_type_attr_matlab, "cell")
+        end
     finally
         close(ctype)
         close(cset)
@@ -680,8 +687,34 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::M
     end
 end
 
+# Write empty (zero-dimensional) structs with no fields
+function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, s::EmptyStruct)
+    dset, dtype = create_dataset(parent, name, s.dims)
+    try
+        write_attribute(dset, empty_attr_matlab, 0x01)
+        write_attribute(dset, name_type_attr_matlab, "struct")
+        write_dataset(dset, dtype, s.dims)
+    finally
+        close(dtype); close(dset)
+    end
+end
+
 # Write a struct from arrays of keys and values
 function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, k::Vector{String}, v::Vector)
+    if length(k) == 0
+        # empty struct
+        adata = UInt64[1, 1]
+        dset, dtype = create_dataset(parent, name, adata)
+        try
+            write_attribute(dset, empty_attr_matlab, 0x01)
+            write_attribute(dset, name_type_attr_matlab, "struct")
+            write_dataset(dset, dtype, adata)
+        finally
+            close(dtype); close(dset)
+        end
+        return
+    end
+
     g = create_group(parent, name)
     try
         write_attribute(g, name_type_attr_matlab, "struct")
@@ -719,11 +752,35 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, dat::D
 end
 
 function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::MatlabOpaque)
-    error("writing of MatlabOpaque types is not yet supported")
+    if obj.class == "FileWrapper__"
+        m_write(mfile, parent, name, obj["__filewrapper__"], UInt32(3))
+        return
+    end
+
+    metadata = MAT_subsys.set_mcos_object_metadata(mfile.subsystem, obj)
+    dset, dtype = create_dataset(parent, name, metadata)
+    try
+        write_dataset(dset, dtype, metadata)
+        write_attribute(dset, name_type_attr_matlab, obj.class)
+        write_attribute(dset, object_type_attr_matlab, UInt32(3))
+    finally
+        close(dset)
+        close(dtype)
+    end
 end
 
 function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::AbstractArray{MatlabOpaque})
-    error("writing of MatlabOpaque types is not yet supported")
+    metadata = MAT_subsys.set_mcos_object_metadata(mfile.subsystem, obj)
+    dset, dtype = create_dataset(parent, name, metadata)
+    try
+        # TODO: Handle empty array case
+        write_dataset(dset, dtype, metadata)
+        write_attribute(dset, name_type_attr_matlab, first(obj).class)
+        write_attribute(dset, object_type_attr_matlab, UInt32(3))
+    finally
+        close(dset)
+        close(dtype)
+    end
 end
 
 function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, arr::PooledArray)
@@ -743,6 +800,11 @@ function write(parent::MatlabHDF5File, name::String, thing)
     m_write(parent, parent.plain, name, thing)
 end
 
+function write_subsys(mfile::MatlabHDF5File, subsys_data::Dict{String,Any})
+    name = "#subsystem#"
+    m_write(mfile, mfile.plain, name, subsys_data)
+end
+
 ## Type conversion operations ##
 
 struct MatlabString end