DateTime and Millisecond writing (#218)

* DateTime and Millisecond writing

* option to avoid automatic MatlabOpaque conversions for troubleshooting

* better isapprox support

Author: matthijscox

src/MAT.jl

@@ -41,11 +41,11 @@ export MatlabStructArray, MatlabClassObject, MatlabOpaque, MatlabTable
 
 # Open a MATLAB file
 const HDF5_HEADER = UInt8[0x89, 0x48, 0x44, 0x46, 0x0d, 0x0a, 0x1a, 0x0a]
-function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool; table::Type=MatlabTable)
+function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool; table::Type=MatlabTable, convert_opaque::Bool=true)
     # When creating new files, create as HDF5 by default
     fs = filesize(filename)
     if cr && (tr || fs == 0)
-        return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, Base.ENDIAN_BOM == 0x04030201; table=table)
+        return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, Base.ENDIAN_BOM == 0x04030201; table=table, convert_opaque=convert_opaque)
     elseif fs == 0
         error("File \"$filename\" does not exist and create was not specified")
     end
@@ -73,26 +73,26 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo
         if wr || cr || tr || ff
             error("creating or appending to MATLAB v5 files is not supported")
         end
-        return MAT_v5.matopen(rawfid, endian_indicator; table=table)
+        return MAT_v5.matopen(rawfid, endian_indicator; table=table, convert_opaque=convert_opaque)
     end
 
     # Check for HDF5 file
     for offset = 512:512:fs-8
         seek(rawfid, offset)
         if read!(rawfid, Vector{UInt8}(undef, 8)) == HDF5_HEADER
             close(rawfid)
-            return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, endian_indicator == 0x494D; table=table)
+            return MAT_HDF5.matopen(filename, rd, wr, cr, tr, ff, compress, endian_indicator == 0x494D; table=table, convert_opaque=convert_opaque)
         end
     end
 
     close(rawfid)
     error("\"$filename\" is not a MAT file")
 end
 
-function matopen(fname::AbstractString, mode::AbstractString; compress::Bool = false, table::Type = MatlabTable)
-    mode == "r"  ? matopen(fname, true , false, false, false, false, false; table=table)    :
-    mode == "r+" ? matopen(fname, true , true , false, false, false, compress; table=table) :
-    mode == "w"  ? matopen(fname, false, true , true , true , false, compress; table=table) :
+function matopen(fname::AbstractString, mode::AbstractString; compress::Bool = false, kwargs...)
+    mode == "r"  ? matopen(fname, true , false, false, false, false, false; kwargs...)    :
+    mode == "r+" ? matopen(fname, true , true , false, false, false, compress; kwargs...) :
+    mode == "w"  ? matopen(fname, false, true , true , true , false, compress; kwargs...) :
     # mode == "w+" ? matopen(fname, true , true , true , true , false, compress) :
     # mode == "a"  ? matopen(fname, false, true , true , false, true, compress)  :
     # mode == "a+" ? matopen(fname, true , true , true , false, true, compress)  :
@@ -185,8 +185,8 @@ vars["s"]["testTable"]
    3 │    3489.0  Smith     2016-12-22T00:00:00  Fair    Late, but only by half an hour. …
 ```
 """
-function matread(filename::AbstractString; table::Type=MatlabTable)
-    file = matopen(filename; table=table)
+function matread(filename::AbstractString; table::Type=MatlabTable, convert_opaque::Bool=true)
+    file = matopen(filename; table=table, convert_opaque=convert_opaque)
     local vars
     try
         vars = read(file)

src/MAT_HDF5.jl

@@ -36,7 +36,16 @@ import HDF5: Reference
 import Dates
 import Tables
 import PooledArrays: PooledArray
-import ..MAT_types: MatlabStructArray, StructArrayField, convert_struct_array, MatlabClassObject, MatlabOpaque, MatlabTable, EmptyStruct
+
+import ..MAT_types: 
+    convert_struct_array,
+    EmptyStruct,
+    MatlabClassObject, 
+    MatlabOpaque,
+    MatlabStructArray,  
+    MatlabTable, 
+    ScalarOrArray,
+    StructArrayField
 
 const HDF5Parent = Union{HDF5.File, HDF5.Group}
 const HDF5BitsOrBool = Union{HDF5.BitsType,Bool}
@@ -100,7 +109,7 @@ function close(f::MatlabHDF5File)
     nothing
 end
 
-function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool, endian_indicator::Bool; table::Type=MatlabTable)
+function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool, ff::Bool, compress::Bool, endian_indicator::Bool; table::Type=MatlabTable, convert_opaque::Bool=true)
     local f
     if ff && !wr
         error("Cannot append to a read-only file")
@@ -132,6 +141,7 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo
     subsys_refs = "#subsystem#"
     if rd && haskey(fid.plain, subsys_refs)
         fid.subsystem.table_type = table
+        fid.subsystem.convert_opaque = convert_opaque
         subsys_data = m_read(fid.plain[subsys_refs], fid.subsystem)
         MAT_subsys.load_subsys!(fid.subsystem, subsys_data, endian_indicator)
     elseif wr
@@ -747,8 +757,12 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, s)
     m_write(mfile, parent, name, check_struct_keys([string(x) for x in fieldnames(T)]), [getfield(s, x) for x in fieldnames(T)])
 end
 
-function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, dat::Dates.AbstractTime)
-    error("writing of Dates types is not yet supported")
+function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, dat::ScalarOrArray{T}) where T<:Dates.AbstractTime
+    error("writing of type $T is not yet supported")
+end
+
+function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, dat::ScalarOrArray{T}) where T<:Union{Dates.DateTime, Dates.Millisecond}
+    m_write(mfile, parent, name, MatlabOpaque(dat))
 end
 
 function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::MatlabOpaque)

src/MAT_subsys.jl

@@ -57,7 +57,10 @@ mutable struct Subsystem
     prop_vals_defaults::Any
     handle_data::Any
     java_data::Any
-    table_type::Type # Julia type to convert Matlab tables into
+
+    # automatic MatlabOpaque conversion
+    convert_opaque::Bool
+    table_type::Type 
 
     # Counters for saving
     saveobj_counter::UInt32
@@ -84,6 +87,7 @@ mutable struct Subsystem
             nothing,
             nothing,
             nothing,
+            true,
             Nothing,
             UInt32(0),
             UInt32(0),
@@ -420,7 +424,11 @@ function load_mcos_object(metadata::Array{UInt32}, type_name::String, subsys::Su
     if nobjects == 1
         oid = object_ids[1]
         obj = get_object!(subsys, oid, classname)
-        return convert_opaque(obj; table=subsys.table_type)
+        if subsys.convert_opaque
+            return convert_opaque(obj; table=subsys.table_type)
+        else
+            return obj
+        end
     else
         # no need to convert_opaque, matlab wraps object arrays in a single class normally
         object_arr = Array{MatlabOpaque}(undef, convert(Vector{Int}, dims)...)

src/MAT_types.jl

@@ -30,6 +30,7 @@ module MAT_types
 
 using StringEncodings: StringEncodings
 import StringEncodings: Encoding
+import Dates
 import Dates: DateTime, Second, Millisecond
 import PooledArrays: PooledArray, RefArray
 using Tables: Tables
@@ -38,6 +39,9 @@ export MatlabStructArray, StructArrayField, convert_struct_array
 export MatlabClassObject
 export MatlabOpaque, convert_opaque
 export MatlabTable
+export ScalarOrArray
+
+const ScalarOrArray{T} = Union{T, AbstractArray{T}}
 
 # struct arrays are stored as columns per field name
 """
@@ -169,7 +173,7 @@ function Base.:(==)(m1::MatlabStructArray{N}, m2::MatlabStructArray{N}) where {N
 end
 
 function Base.isapprox(m1::MatlabStructArray, m2::MatlabStructArray; kwargs...)
-    return isequal(m1.names, m2.names) && isapprox(m1.values, m2.values; kwargs...)
+    return isequal(m1.class, m2.class) && isequal(m1.names, m2.names) && isapprox(m1.values, m2.values; kwargs...)
 end
 
 function find_index(m::MatlabStructArray, s::AbstractString)
@@ -308,6 +312,14 @@ Base.iterate(m::MatlabClassObject) = iterate(m.d)
 Base.haskey(m::MatlabClassObject, k) = haskey(m.d, k)
 Base.get(m::MatlabClassObject, k, default) = get(m.d, k, default)
 
+function Base.:(==)(m1::MatlabClassObject, m2::MatlabClassObject)
+    return m1.class == m2.class && m1.d == m2.d
+end
+
+function Base.isapprox(m1::MatlabClassObject, m2::MatlabClassObject; kwargs...)
+    return m1.class == m2.class && dict_isapprox(m1.d, m2.d; kwargs...)
+end
+
 function MatlabStructArray(arr::AbstractArray{MatlabClassObject})
     first_obj, remaining_obj = Iterators.peel(arr)
     class = first_obj.class
@@ -374,6 +386,28 @@ Base.iterate(m::MatlabOpaque) = iterate(m.d)
 Base.haskey(m::MatlabOpaque, k) = haskey(m.d, k)
 Base.get(m::MatlabOpaque, k, default) = get(m.d, k, default)
 
+function Base.:(==)(m1::MatlabOpaque, m2::MatlabOpaque)
+    return m1.class == m2.class && m1.d == m2.d
+end
+
+function Base.isapprox(m1::MatlabOpaque, m2::MatlabOpaque; kwargs...)
+    return m1.class == m2.class && dict_isapprox(m1.d, m2.d; kwargs...)
+end
+
+function dict_isapprox(d1::AbstractDict{T}, d2::AbstractDict{T}; kwargs...) where T
+    keys(d1) == keys(d2) || return false
+    for k in keys(d1)
+        v1, v2 = d1[k], d2[k]
+        value_isapprox(v1, v2) || return false
+    end
+    return true
+end
+dict_isapprox(d1::AbstractDict{T1}, d2::AbstractDict{T2}; kwargs...) where {T1,T2} = false
+
+value_isapprox(x1::AbstractString, x2::AbstractString; kwargs...) = isequal(x1, x2)
+value_isapprox(x1, x2; kwargs...) = isapprox(x1, x2; kwargs...)
+value_isapprox(d1::AbstractDict, d2::AbstractDict; kwargs...) = dict_isapprox(x1, x2; kwargs...)
+
 function convert_opaque(obj::MatlabOpaque; table::Type=Nothing)
     if obj.class == "string"
         return from_string(obj)
@@ -445,6 +479,28 @@ function ms_to_datetime(ms::Real)
     return DateTime(1970, 1, 1) + Second(s) + Millisecond(ms_rem)
 end
 
+function to_matlab_data(d::DateTime)
+    ms = Dates.value(d)
+
+    # matlab counts w.r.t. 1970
+    matlab_offset = Dates.value(DateTime(1970, 1, 1))
+
+    # note: can be negative, that's fine
+    matlab_ms = ms - matlab_offset
+    return Complex(Float64(matlab_ms))
+end
+
+function MatlabOpaque(d::ScalarOrArray{DateTime})
+    return MatlabOpaque(
+        Dict(
+            "tz"   => "",
+            "data" => map_or_not(to_matlab_data, d),
+            "fmt"  => "",
+        ),
+        "datetime"
+    )
+end
+
 function from_duration(obj::MatlabOpaque)
     dat = obj["millis"]
     #fmt = obj["fmt"] # TODO: format, e.g. 'd' to Day
@@ -454,6 +510,17 @@ function from_duration(obj::MatlabOpaque)
     return map_or_not(Millisecond, dat)
 end
 
+to_matlab_millis(d::Millisecond) = Float64(Dates.value(d))
+
+function MatlabOpaque(d::ScalarOrArray{Millisecond})
+    return MatlabOpaque(
+        Dict(
+            "millis" => map_or_not(to_matlab_millis, d),
+        ),
+        "duration"
+    )
+end
+
 function from_categorical(obj::MatlabOpaque)
     category_names = obj["categoryNames"]
     codes = obj["codes"]

src/MAT_v5.jl

@@ -394,7 +394,7 @@ function read_matrix(f::IO, swap_bytes::Bool, subsys::Subsystem)
 end
 
 # Open MAT file for reading
-function matopen(ios::IOStream, endian_indicator::UInt16; table::Type=MatlabTable)
+function matopen(ios::IOStream, endian_indicator::UInt16; table::Type=MatlabTable, convert_opaque::Bool=true)
     matfile = Matlabv5File(ios, endian_indicator == 0x494D)
 
     seek(matfile.ios, 116)
@@ -405,6 +405,7 @@ function matopen(ios::IOStream, endian_indicator::UInt16; table::Type=MatlabTabl
     if subsys_offset != 0
         matfile.subsystem_position = subsys_offset
         matfile.subsystem.table_type = table
+        matfile.subsystem.convert_opaque = convert_opaque
         read_subsystem!(matfile)
     end
 

test/read.jl

@@ -1,5 +1,6 @@
 using MAT, Test
 using Dates
+using SparseArrays, LinearAlgebra
 
 function check(filename, result)
     matfile = matopen(filename)
@@ -271,6 +272,12 @@ for format in ["v7", "v7.3"]
         dt = vars["testDatetime"]
         @test dt isa DateTime
         @test dt - DateTime(2019, 12, 2, 16, 42, 49) < Second(1)
+
+        # test no conversion at all
+        vars = matread(filepath; convert_opaque=false)["s"]
+        t = vars["testTable"]
+        @test t isa MatlabOpaque
+        @test t["data"][3] isa MatlabOpaque
     end
     end