From 7710b78ee169abcd5d9fdd959bf8be4a42ab10ae Mon Sep 17 00:00:00 2001 From: Nathan West Date: Sat, 11 Apr 2020 18:58:01 -0400 Subject: [PATCH 1/4] move headers to an include directory --- CMakeLists.txt | 10 +++++----- examples/example_reading_sigmf_file.cpp | 4 ++-- examples/example_record_with_multiple_namespaces.cpp | 3 ++- examples/example_record_with_variadic_dataclass.cpp | 2 +- examples/example_sigmf_json_roundtrip.cpp | 3 ++- src/{ => include/libsigmf}/annotation.h | 0 src/{ => include/libsigmf}/capture.h | 0 .../libsigmf}/flatbuffers_json_visitor.h | 12 ++++-------- .../libsigmf}/flatbuffers_type_to_json.h | 0 src/{ => include/libsigmf}/global.h | 0 src/include/libsigmf/json_wrap.h | 4 ++++ src/{ => include/libsigmf}/sigmf.h | 2 -- src/{ => include/libsigmf}/sigmf_helpers.h | 0 src/{ => include/libsigmf}/variadic_data_class.h | 2 ++ 14 files changed, 22 insertions(+), 20 deletions(-) rename src/{ => include/libsigmf}/annotation.h (100%) rename src/{ => include/libsigmf}/capture.h (100%) rename src/{ => include/libsigmf}/flatbuffers_json_visitor.h (98%) rename src/{ => include/libsigmf}/flatbuffers_type_to_json.h (100%) rename src/{ => include/libsigmf}/global.h (100%) create mode 100644 src/include/libsigmf/json_wrap.h rename src/{ => include/libsigmf}/sigmf.h (98%) rename src/{ => include/libsigmf}/sigmf_helpers.h (100%) rename src/{ => include/libsigmf}/variadic_data_class.h (99%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f58c73..8c554d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,7 +120,7 @@ set(LIBSIGMF_GEN_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/sigmf_protocols") add_library(libsigmf INTERFACE) target_include_directories(libsigmf INTERFACE $ - $ + $ $ $ ) @@ -176,15 +176,15 @@ configure_file( ######################################################################## install( # install flatbuf proto defs DIRECTORY ${LIBSIGMF_GEN_HEADERS}/ - DESTINATION include/sigmf/fbs + DESTINATION include/libsigmf/fbs FILES_MATCHING PATTERN "*.fbs)") install( # install generated headers DIRECTORY ${LIBSIGMF_GEN_HEADERS}/ - DESTINATION include/sigmf + DESTINATION include/libsigmf FILES_MATCHING PATTERN "*.h") install( # install original headers - DIRECTORY ${CMAKE_SOURCE_DIR}/src/ - DESTINATION include/sigmf + DIRECTORY ${CMAKE_SOURCE_DIR}/src/include + DESTINATION include/ FILES_MATCHING PATTERN "*.h") ######################################################################## diff --git a/examples/example_reading_sigmf_file.cpp b/examples/example_reading_sigmf_file.cpp index 3bf3cfb..a3fef25 100644 --- a/examples/example_reading_sigmf_file.cpp +++ b/examples/example_reading_sigmf_file.cpp @@ -16,8 +16,8 @@ #include "sigmf_core_generated.h" #include "sigmf_antenna_generated.h" -#include "sigmf.h" -#include "sigmf_helpers.h" +#include "libsigmf/sigmf.h" +#include "libsigmf/sigmf_helpers.h" #include #include diff --git a/examples/example_record_with_multiple_namespaces.cpp b/examples/example_record_with_multiple_namespaces.cpp index f6b078d..e75b165 100644 --- a/examples/example_record_with_multiple_namespaces.cpp +++ b/examples/example_record_with_multiple_namespaces.cpp @@ -16,7 +16,8 @@ #include "sigmf_core_generated.h" #include "sigmf_antenna_generated.h" -#include "sigmf.h" +#include "libsigmf/sigmf.h" +#include int main() { diff --git a/examples/example_record_with_variadic_dataclass.cpp b/examples/example_record_with_variadic_dataclass.cpp index 4b40e66..87e2beb 100644 --- a/examples/example_record_with_variadic_dataclass.cpp +++ b/examples/example_record_with_variadic_dataclass.cpp @@ -16,7 +16,7 @@ #include "sigmf_core_generated.h" -#include "sigmf.h" +#include "libsigmf/sigmf.h" #include int main() { diff --git a/examples/example_sigmf_json_roundtrip.cpp b/examples/example_sigmf_json_roundtrip.cpp index 25927f0..1139d9f 100644 --- a/examples/example_sigmf_json_roundtrip.cpp +++ b/examples/example_sigmf_json_roundtrip.cpp @@ -14,8 +14,9 @@ * limitations under the License. */ -#include "sigmf.h" +#include "libsigmf/sigmf.h" #include +#include int main() { diff --git a/src/annotation.h b/src/include/libsigmf/annotation.h similarity index 100% rename from src/annotation.h rename to src/include/libsigmf/annotation.h diff --git a/src/capture.h b/src/include/libsigmf/capture.h similarity index 100% rename from src/capture.h rename to src/include/libsigmf/capture.h diff --git a/src/flatbuffers_json_visitor.h b/src/include/libsigmf/flatbuffers_json_visitor.h similarity index 98% rename from src/flatbuffers_json_visitor.h rename to src/include/libsigmf/flatbuffers_json_visitor.h index 1c310ad..f56b123 100644 --- a/src/flatbuffers_json_visitor.h +++ b/src/include/libsigmf/flatbuffers_json_visitor.h @@ -17,13 +17,9 @@ #ifndef LIBSIGMF_FLATBUFFERS_TO_JSON_VISITOR_H #define LIBSIGMF_FLATBUFFERS_TO_JSON_VISITOR_H -#include "variadic_data_class.h" +#include "json_wrap.h" #include #include -#include -#include - -using json = nlohmann::json; /* @@ -41,7 +37,6 @@ struct FromSigMFVisitor : public flatbuffers::IterationVisitor { flatbuffers::voffset_t last_offset; flatbuffers::Offset last_fb_offset; - explicit FromSigMFVisitor(std::string namespace_prefix, const json &j) : p(namespace_prefix), last_offset(0), _start(0), _stop(0) { fbb = flatbuffers::FlatBufferBuilder(); @@ -124,12 +119,13 @@ struct FromSigMFVisitor : public flatbuffers::IterationVisitor { } void StartVector() override { - std::cout << "the visitor was called on startvector... not implemented" << std::endl; + + throw std::runtime_error("the visitor was called on startvector... not implemented"); } void EndVector() override { - std::cout << "the visitor was called on endvector... not implemented" << std::endl; + throw std::runtime_error("the visitor was called on endvector... not implemented"); } void Element(size_t i, flatbuffers::ElementaryType /*type*/, diff --git a/src/flatbuffers_type_to_json.h b/src/include/libsigmf/flatbuffers_type_to_json.h similarity index 100% rename from src/flatbuffers_type_to_json.h rename to src/include/libsigmf/flatbuffers_type_to_json.h diff --git a/src/global.h b/src/include/libsigmf/global.h similarity index 100% rename from src/global.h rename to src/include/libsigmf/global.h diff --git a/src/include/libsigmf/json_wrap.h b/src/include/libsigmf/json_wrap.h new file mode 100644 index 0000000..f8d2357 --- /dev/null +++ b/src/include/libsigmf/json_wrap.h @@ -0,0 +1,4 @@ + +#include + +using json = nlohmann::json; \ No newline at end of file diff --git a/src/sigmf.h b/src/include/libsigmf/sigmf.h similarity index 98% rename from src/sigmf.h rename to src/include/libsigmf/sigmf.h index 9bc42c3..f5fa24c 100644 --- a/src/sigmf.h +++ b/src/include/libsigmf/sigmf.h @@ -17,13 +17,11 @@ #ifndef LIBSIGMF_SIGMF_H #define LIBSIGMF_SIGMF_H -#include "variadic_data_class.h" #include "global.h" #include "capture.h" #include "annotation.h" #include "sigmf_core_generated.h" #include -#include namespace sigmf { diff --git a/src/sigmf_helpers.h b/src/include/libsigmf/sigmf_helpers.h similarity index 100% rename from src/sigmf_helpers.h rename to src/include/libsigmf/sigmf_helpers.h diff --git a/src/variadic_data_class.h b/src/include/libsigmf/variadic_data_class.h similarity index 99% rename from src/variadic_data_class.h rename to src/include/libsigmf/variadic_data_class.h index 4676318..6b86912 100644 --- a/src/variadic_data_class.h +++ b/src/include/libsigmf/variadic_data_class.h @@ -18,6 +18,8 @@ #define LIBSIGMF_VARIADICDATACLASS_H #include "flatbuffers_json_visitor.h" +#include "json_wrap.h" +#include #include #include #include From c32d8b24a67c8bd4501c10d0ad627731cf3dc2b5 Mon Sep 17 00:00:00 2001 From: Nathan West Date: Sat, 11 Apr 2020 19:44:27 -0400 Subject: [PATCH 2/4] move all of flatbuffervisitor to a static lib --- CMakeLists.txt | 23 +- .../libsigmf/flatbuffers_json_visitor.h | 496 ++---------------- src/include/libsigmf/json_wrap.h | 2 +- src/include/libsigmf/variadic_data_class.h | 2 +- src/lib/flatbuffers_json_visitor.cpp | 488 +++++++++++++++++ src/lib/json_wrap.cpp | 3 + 6 files changed, 555 insertions(+), 459 deletions(-) create mode 100644 src/lib/flatbuffers_json_visitor.cpp create mode 100644 src/lib/json_wrap.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c554d6..94aaafa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,13 +117,20 @@ set(LIBSIGMF_GEN_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/sigmf_protocols") ######################################################################## # Our interface target that downstream have to use ######################################################################## -add_library(libsigmf INTERFACE) -target_include_directories(libsigmf INTERFACE +add_library(libsigmf + STATIC + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib/flatbuffers_json_visitor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib/json_wrap.cpp + ) +target_include_directories(libsigmf + INTERFACE $ $ $ $ -) + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/include + ) if (${Flatbuffers_FOUND}) # System flatbuffers can import this target @@ -131,8 +138,11 @@ if (${Flatbuffers_FOUND}) flatbuffers::flatbuffers_shared ) else() - target_include_directories(libsigmf INTERFACE + target_include_directories(libsigmf + INTERFACE $ + PRIVATE + ${FLATBUFFERS_INCLUDE_DIR} ) endif(${Flatbuffers_FOUND}) @@ -140,8 +150,11 @@ if (${nlohmann_json_FOUND}) # TODO: add target_link_libraries interface to nlohmann json # when we use the system version else() - target_include_directories(libsigmf INTERFACE + target_include_directories(libsigmf + INTERFACE $ + PRIVATE + ${JSON_INCLUDE_DIR} ) endif(${nlohmann_json_FOUND}) diff --git a/src/include/libsigmf/flatbuffers_json_visitor.h b/src/include/libsigmf/flatbuffers_json_visitor.h index f56b123..8f325bb 100644 --- a/src/include/libsigmf/flatbuffers_json_visitor.h +++ b/src/include/libsigmf/flatbuffers_json_visitor.h @@ -30,168 +30,61 @@ struct FromSigMFVisitor : public flatbuffers::IterationVisitor { flatbuffers::uoffset_t _start; flatbuffers::uoffset_t _stop; - json narrowest_json; + std::unique_ptr narrowest_json; std::string p; std::string last_field_name; flatbuffers::FlatBufferBuilder fbb; flatbuffers::voffset_t last_offset; flatbuffers::Offset last_fb_offset; - explicit FromSigMFVisitor(std::string namespace_prefix, const json &j) - : p(namespace_prefix), last_offset(0), _start(0), _stop(0) { - fbb = flatbuffers::FlatBufferBuilder(); - } + explicit FromSigMFVisitor(std::string namespace_prefix, const json &j); - void StartSequence() override { - _start = fbb.StartTable(); - } + void StartSequence() override; - void EndSequence() override { - _stop = fbb.EndTable(_start); - } + void EndSequence() override; void Field(size_t field_idx, size_t set_idx, flatbuffers::ElementaryType e_type, bool is_vector, const flatbuffers::TypeTable *type_table, - const char *name, const uint8_t *val, json jj) { - narrowest_json = jj; - last_field_name = name; - last_offset = flatbuffers::FieldIndexToOffset(static_cast(field_idx)); - } + const char *name, const uint8_t *val, json jj); template - void Named(T x, const char *name) { - try { - fbb.AddElement(last_offset, narrowest_json.at(p + last_field_name).get(), T{}); - } catch (nlohmann::detail::out_of_range &e) { - } - } + void Named(T x, const char *name); - void UType(uint8_t x, const char *name) override { Named(x, name); } + void UType(uint8_t x, const char *name) override; // void Bool(bool x) override { s += x ? "true" : "false"; } - void Char(int8_t x, const char *name) override { Named(x, name); } - - void UChar(uint8_t x, const char *name) override { Named(x, name); } - - void Short(int16_t x, const char *name) override { Named(x, name); } - - void UShort(uint16_t x, const char *name) override { Named(x, name); } - - void Int(int32_t x, const char *name) override { Named(x, name); } - - void UInt(uint32_t x, const char *name) override { Named(x, name); } - - void Long(int64_t x) override { - try { - fbb.AddElement(last_offset, narrowest_json.at(p + last_field_name).get(), int64_t(0)); - } catch (nlohmann::detail::out_of_range &e) { - } - } - - void ULong(uint64_t x) override { - try { - fbb.AddElement(last_offset, narrowest_json.at(p + last_field_name).get(), uint64_t(0)); - } catch (nlohmann::detail::out_of_range &e) { - } - } - - void Float(float x) override { - try { - fbb.AddElement(last_offset, narrowest_json.at(p + last_field_name).get(), 0.f); - } catch (nlohmann::detail::out_of_range &e) { - } - } - - void Double(double x) override { - try { - fbb.AddElement(last_offset, narrowest_json.at(p + last_field_name).get(), 0.0); - } catch (nlohmann::detail::out_of_range &e) { - } - } - - void String(const struct flatbuffers::String *str) override { - try { - auto strval = narrowest_json.at(p + last_field_name).get(); - last_fb_offset = flatbuffers::Offset(fbb.CreateString(strval).o); - } catch (nlohmann::detail::out_of_range &e) { - last_fb_offset.o = 0; - }; - } - - void StartVector() override { - - throw std::runtime_error("the visitor was called on startvector... not implemented"); - - } - - void EndVector() override { - throw std::runtime_error("the visitor was called on endvector... not implemented"); - } - - void Element(size_t i, flatbuffers::ElementaryType /*type*/, - const flatbuffers::TypeTable * /*type_table*/, const uint8_t * /*val*/) override {} -}; + virtual void Char(int8_t x, const char *name) override; + + virtual void UChar(uint8_t x, const char *name) override; + + virtual void Short(int16_t x, const char *name) override; + + virtual void UShort(uint16_t x, const char *name) override; + + virtual void Int(int32_t x, const char *name) override; + + virtual void UInt(uint32_t x, const char *name) override; + + virtual void Long(int64_t x) override; + virtual void ULong(uint64_t x) override; -template -flatbuffers::Offset json_vector_to_flatbuffer(flatbuffers::FlatBufferBuilder &fbb, const json &jvec) { - size_t dtype_size = sizeof(T); - std::vector tmpvec; - tmpvec.reserve(jvec.size()); - for (auto elem = jvec.begin(); elem != jvec.end(); ++elem) { - tmpvec.emplace_back(elem->get()); - } - return flatbuffers::Offset(fbb.CreateVector(tmpvec).o); - -} - -inline flatbuffers::Offset -json_vector_to_chararray(flatbuffers::FlatBufferBuilder &fbb, const json &jvec, flatbuffers::ElementaryType type) { - size_t dtype_size; - size_t num_elements = jvec.size(); - switch (type) { - case flatbuffers::ET_UTYPE: { - auto rvec = jvec.get >(); - return flatbuffers::Offset(fbb.CreateVector(rvec).o); - } - case flatbuffers::ET_BOOL: { - auto rvec = jvec.get >(); - return flatbuffers::Offset(fbb.CreateVector(rvec).o); - } - case flatbuffers::ET_CHAR: { - auto rvec = jvec.get >(); - return flatbuffers::Offset(fbb.CreateVector(rvec).o); - } - case flatbuffers::ET_UCHAR: { - auto rvec = jvec.get >(); - return flatbuffers::Offset(fbb.CreateVector(rvec).o); - } - case flatbuffers::ET_SHORT: - return json_vector_to_flatbuffer(fbb, jvec); - case flatbuffers::ET_USHORT: { - return json_vector_to_flatbuffer(fbb, jvec); - } - case flatbuffers::ET_INT: - return json_vector_to_flatbuffer(fbb, jvec); - case flatbuffers::ET_UINT: { - return json_vector_to_flatbuffer(fbb, jvec); - } - case flatbuffers::ET_LONG: - return json_vector_to_flatbuffer(fbb, jvec); - case flatbuffers::ET_ULONG: { - return json_vector_to_flatbuffer(fbb, jvec); - } - case flatbuffers::ET_FLOAT: { - return json_vector_to_flatbuffer(fbb, jvec); - } - case flatbuffers::ET_DOUBLE: { - return json_vector_to_flatbuffer(fbb, jvec); - } - default: - throw std::runtime_error("libsigmf cannot make a vector of this type yet"); - } -} + virtual void Float(float x) override; + virtual void Double(double x) override; + + virtual void String(const struct flatbuffers::String *str) override; + + virtual void StartVector() override; + + virtual void EndVector() override; + + virtual void Element(size_t i, flatbuffers::ElementaryType /*type*/, + const flatbuffers::TypeTable * /*type_table*/, const uint8_t * /*val*/) override; +}; + +flatbuffers::Offset +json_vector_to_chararray(flatbuffers::FlatBufferBuilder &fbb, const json &jvec, flatbuffers::ElementaryType type); /** * Iterate through a typetable-- I'll be honest here. This is kind of bullshit. We need to create all of @@ -205,265 +98,19 @@ json_vector_to_chararray(flatbuffers::FlatBufferBuilder &fbb, const json &jvec, * @param type_table the table to iterate over * @param visitor the visitor responsible for creating objects and adding fields to its internal flatbufferbuilder */ -inline void IterateType(const flatbuffers::TypeTable *type_table, FromSigMFVisitor *visitor, json original_json) { - const uint8_t *prev_val = nullptr; - const uint8_t val[8] = {}; - auto comosite_type_offsets = std::map >(); - // Generate the strings up front (because they must be created before our Table in the flatbuffer) - for (size_t i = 0; i < type_table->num_elems; i++) { - auto type_code = type_table->type_codes[i]; - auto type = static_cast(type_code.base_type); - auto is_vector = type_code.is_vector != 0; - auto ref_idx = type_code.sequence_ref; - const flatbuffers::TypeTable *ref = nullptr; - if (ref_idx >= 0) { ref = type_table->type_refs[ref_idx](); } - auto name = type_table->names ? type_table->names[i] : nullptr; - // The thing is, at least on this iteration we don't care about making real elementary types, just the - // ones that need an offset - if (type < flatbuffers::ET_STRING) { - if (is_vector) { - auto vector_offset = json_vector_to_chararray(visitor->fbb, original_json[name], type); - auto void_offset = flatbuffers::Offset(vector_offset.o); - comosite_type_offsets[i] = void_offset; - } - } else if (type == flatbuffers::ET_STRING) { - if (is_vector) { - std::vector > vecofstrings; - - for (const auto &item : original_json[visitor->p + name]) { - auto strval = item.get(); - auto last_fb_offset = flatbuffers::Offset(visitor->fbb.CreateString(strval).o); - - vecofstrings.emplace_back(last_fb_offset); - } - auto vecoffset = flatbuffers::Offset(visitor->fbb.CreateVector(vecofstrings).o); - comosite_type_offsets[i] = vecoffset; - } else { - if (!original_json[visitor->p + name].is_null()) { - visitor->Field(i, 0, type, is_vector, ref, name, val, original_json); - IterateValue(type, val, ref, prev_val, -1, visitor); - comosite_type_offsets[i] = visitor->last_fb_offset; - } - } - } else if (type == flatbuffers::ET_SEQUENCE) { - if (is_vector) { - std::vector > vecofstrings; - - for (const auto &item : original_json[visitor->p + name]) { - auto seq_typetable = type_table->type_refs[ref_idx](); - IterateType(seq_typetable, visitor, item); - auto vecelement = flatbuffers::Offset(visitor->_stop); - - vecofstrings.emplace_back(vecelement); - } - auto vecoffset = flatbuffers::Offset(visitor->fbb.CreateVector(vecofstrings).o); - comosite_type_offsets[i] = vecoffset; - } else { - auto seq_typetable = type_table->type_refs[ref_idx](); - - IterateType(seq_typetable, visitor, original_json[name]); - comosite_type_offsets[i] = flatbuffers::Offset(visitor->_stop); - } - } - - } - // Now make the table and fill in all of our types - size_t set_idx = 0; - visitor->StartSequence(); - for (size_t i = 0; i < type_table->num_elems; i++) { - auto type_code = type_table->type_codes[i]; - auto type = static_cast(type_code.base_type); - auto is_vector = type_code.is_vector != 0; - auto ref_idx = type_code.sequence_ref; - const flatbuffers::TypeTable *ref = nullptr; - if (ref_idx >= 0) { ref = type_table->type_refs[ref_idx](); } - auto name = type_table->names ? type_table->names[i] : nullptr; - if (is_vector) { - auto this_voffset = flatbuffers::FieldIndexToOffset(static_cast(i)); - visitor->fbb.AddOffset(this_voffset, comosite_type_offsets.at(i)); - } else if (type == flatbuffers::ET_STRING) { - try { - auto this_voffset = flatbuffers::FieldIndexToOffset(static_cast(i)); - visitor->fbb.AddOffset(this_voffset, comosite_type_offsets.at(i)); - } catch (std::out_of_range &e) { - // It's ok... this is just because a field wasn't present in json so we never had to deserialize - } - } else if (type == flatbuffers::ET_SEQUENCE) { - auto this_voffset = flatbuffers::FieldIndexToOffset(static_cast(i)); - visitor->fbb.AddOffset(this_voffset, comosite_type_offsets.at(i)); - } else { - visitor->Field(i, set_idx, type, is_vector, ref, name, val, original_json); - set_idx++; - IterateValue(type, val, ref, prev_val, -1, visitor); - } - } - visitor->EndSequence(); -} - +void IterateType(const flatbuffers::TypeTable *type_table, FromSigMFVisitor *visitor, json original_json); // forward declare so we can create objects as fields -inline json +json FlatBufferToJson(const uint8_t *buffer_root, const flatbuffers::TypeTable *typetable, const std::string &ns_prefix, bool include_defaults = false); -inline json -flatbuffer_field_to_json(const uint8_t *val, flatbuffers::ElementaryType type, +json +flatbuffer_field_to_json(const uint8_t *val, + flatbuffers::ElementaryType type, const flatbuffers::TypeTable *tt = nullptr, const std::string &ns_prefix = "", - bool include_defaults = false) { - switch (type) { - case flatbuffers::ET_UTYPE: { - uint8_t tval = 0; - if (val) { - tval = *reinterpret_cast(val); - } - return json(tval); - break; - } - case flatbuffers::ET_BOOL: { - uint8_t tval = 0; - if (val) { - tval = json(*reinterpret_cast(val) != 0); - } - return json(tval); - break; - } - case flatbuffers::ET_CHAR: { - int8_t tval = 0; - if (val) { - tval = *reinterpret_cast(val); - } - return json(tval); - break; - } - case flatbuffers::ET_UCHAR: { - uint8_t tval = 0; - if (val) { - tval = *reinterpret_cast(val); - } - return json(tval); - break; - } - case flatbuffers::ET_SHORT: { - int16_t tval =0; - if (val) { - tval = *reinterpret_cast(val); - } - return json(tval); - break; - } - case flatbuffers::ET_USHORT: { - uint16_t tval = 0; - if (val) { - tval = *reinterpret_cast(val); - } - return json(tval); - break; - } - case flatbuffers::ET_INT: { - auto tval = 0; - if (val) { - tval = *reinterpret_cast(val); - } - return json(tval); - break; - } - case flatbuffers::ET_UINT: { - auto tval = 0UL; - if (val) { - tval = *reinterpret_cast(val); - } - return json(tval); - break; - } - case flatbuffers::ET_LONG: { - auto tval = 0LL; - if (val) { - tval = *reinterpret_cast(val); - } - return json(tval); - break; - } - case flatbuffers::ET_ULONG: { - auto tval = 0ULL; - if (val) { - tval = *reinterpret_cast(val); - } - return json(tval); - break; - } - case flatbuffers::ET_FLOAT: { - auto tval = 0.0; - if (val) { - tval = *reinterpret_cast(val); - } - return json(tval); - break; - } - case flatbuffers::ET_DOUBLE: { - auto tval = 0.0; - if (val) { - tval = *reinterpret_cast(val); - } - return json(tval); - break; - } - case flatbuffers::ET_STRING: { - std::string tval{}; - if (val) { - val += flatbuffers::ReadScalar(val); - tval = reinterpret_cast(val)->c_str(); - } - return json(tval); - break; - } - case flatbuffers::ET_SEQUENCE: { - switch (tt->st) { - case flatbuffers::ST_TABLE: - val += flatbuffers::ReadScalar(val); - return FlatBufferToJson(val, tt, ns_prefix, include_defaults); - // Have not implemented structs, unions, or enums which are all supported by flatbuffers schema - // and would be great to have, but dont make as much sense in json-land. They're on the todo list... - // here is what flatbuffers does internally (see IterateValue in minireflect.h) - default: - throw std::runtime_error( - "libsigmf has not implemented flatbuffer struct, union, or enum to json yet"); -// case flatbuffers::ST_STRUCT: IterateObject(val, type_table, visitor); break; -// case flatbuffers::ST_UNION: { -// val += flatbuffers::ReadScalar(val); -// FLATBUFFERS_ASSERT(prev_val); -// auto union_type = *prev_val; // Always a uint8_t. -// if (vector_index >= 0) { -// auto type_vec = reinterpret_cast *>(prev_val); -// union_type = type_vec->Get(static_cast(vector_index)); -// } -// auto type_code_idx = -// flatbuffers::LookupEnum(union_type, type_table->values, type_table->num_elems); -// if (type_code_idx >= 0 && -// type_code_idx < static_cast(type_table->num_elems)) { -// auto type_code = type_table->type_codes[type_code_idx]; -// switch (type_code.base_type) { -// case flatbuffers::ET_SEQUENCE: { -// auto ref = type_table->type_refs[type_code.sequence_ref](); -// IterateObject(val, ref, visitor); -// break; -// } -// case flatbuffers::ET_STRING: -// visitor->String(reinterpret_cast(val)); -// break; -// default: visitor->Unknown(val); -// } -// } else { -// visitor->Unknown(val); -// } -// break; -// } -// case flatbuffers::ST_ENUM: FLATBUFFERS_ASSERT(false); break; - - } - } - } -} + bool include_defaults = false); /** @@ -477,64 +124,9 @@ flatbuffer_field_to_json(const uint8_t *val, flatbuffers::ElementaryType type, * if there is a value. If so, serialize it to json and shove its value inside the json_object we want to return * using the name of the field as the key. */ -inline json +json FlatBufferToJson(const uint8_t *buffer_root, const flatbuffers::TypeTable *typetable, const std::string &ns_prefix, - bool include_defaults) { - json json_object; - const auto obj = reinterpret_cast(buffer_root); - for (size_t i = 0; i < typetable->num_elems; i++) { - // Gather all of the underlying info about this element in table - auto type_code = typetable->type_codes[i]; - auto type = static_cast(type_code.base_type); - auto is_vector = type_code.is_vector != 0; - auto ref_idx = type_code.sequence_ref; - const flatbuffers::TypeTable *ref = nullptr; - if (ref_idx >= 0) { ref = typetable->type_refs[ref_idx](); } - auto name = typetable->names ? typetable->names[i] : nullptr; - const uint8_t *val = nullptr; - - // Fetch the actual value of this field now and stick it inside our json object - if (typetable->st == flatbuffers::ST_TABLE) { - auto field_offset = flatbuffers::FieldIndexToOffset(static_cast(i)); - val = obj->GetAddressOf(field_offset); - } else { - // when does this get triggered? Probably when someone tries to pass a non-flattbuffers object - // to us. Let's throw an exception so it'll be nice and obvious with a good test-case when this - // happens. - throw std::runtime_error( - "libsigmf::FlatBufferToJson was called with a non-table type. Please file an issue with your fbs schema and input data."); - // Here is what flatbuffers does internally: val = obj + typetable->values[i]; - } - const flatbuffers::TypeTable *ttptr = nullptr; - if (val) { - if (ref_idx >= 0) { - ttptr = typetable->type_refs[ref_idx](); - } - if (is_vector) { - val += flatbuffers::ReadScalar(val); - auto vec = reinterpret_cast *>(val); - auto elem_ptr = vec->Data(); - for (size_t j = 0; j < vec->size(); j++) { - json_object[ns_prefix + name].push_back( - flatbuffer_field_to_json(elem_ptr, type, ttptr, ns_prefix, include_defaults)); - elem_ptr += InlineSize(type, ref); - } - } else { - json lval = flatbuffer_field_to_json(val, type, ttptr, ns_prefix, include_defaults); - json_object[ns_prefix + name] = lval; - } - } else { - // There is no value in this field. This is OK except maybe we want to figure out how to enforce - // required fields. Although maybe flatbuffers schema language and the flatbuffersbuilders have - // already enforced such things. - } - if (!is_vector && type < flatbuffers::ET_SEQUENCE && include_defaults) { - json lval = flatbuffer_field_to_json(val, type, ttptr, ns_prefix, include_defaults); - json_object[ns_prefix + name] = lval; - } - } - return json_object; -} + bool include_defaults); #endif //LIBSIGMF_FLATBUFFERS_TO_JSON_VISITOR_H diff --git a/src/include/libsigmf/json_wrap.h b/src/include/libsigmf/json_wrap.h index f8d2357..d4f68ac 100644 --- a/src/include/libsigmf/json_wrap.h +++ b/src/include/libsigmf/json_wrap.h @@ -1,4 +1,4 @@ -#include +#include using json = nlohmann::json; \ No newline at end of file diff --git a/src/include/libsigmf/variadic_data_class.h b/src/include/libsigmf/variadic_data_class.h index 6b86912..20ccd18 100644 --- a/src/include/libsigmf/variadic_data_class.h +++ b/src/include/libsigmf/variadic_data_class.h @@ -19,6 +19,7 @@ #include "flatbuffers_json_visitor.h" #include "json_wrap.h" +#include #include #include #include @@ -27,7 +28,6 @@ namespace sigmf { - template struct SameType { static const bool value = false; diff --git a/src/lib/flatbuffers_json_visitor.cpp b/src/lib/flatbuffers_json_visitor.cpp new file mode 100644 index 0000000..18db45a --- /dev/null +++ b/src/lib/flatbuffers_json_visitor.cpp @@ -0,0 +1,488 @@ + + +#include "libsigmf/flatbuffers_json_visitor.h" +#include +#include "json_wrap.cpp" + + +template +flatbuffers::Offset json_vector_to_flatbuffer(flatbuffers::FlatBufferBuilder &fbb, json jvec) { + size_t dtype_size = sizeof(T); + std::vector tmpvec; + tmpvec.reserve(jvec.size()); + for (auto elem = jvec.begin(); elem != jvec.end(); ++elem) { + tmpvec.emplace_back(elem->get()); + } + return flatbuffers::Offset(fbb.CreateVector(tmpvec).o); +} + +FromSigMFVisitor::FromSigMFVisitor(std::string namespace_prefix, const json &j) + : p(namespace_prefix), last_offset(0), _start(0), _stop(0) { + fbb = flatbuffers::FlatBufferBuilder(); + narrowest_json = std::make_unique(); + *narrowest_json = j; +} + +void FromSigMFVisitor::StartSequence() { + _start = fbb.StartTable(); +} + +void FromSigMFVisitor::EndSequence() { + _stop = fbb.EndTable(_start); +} + +void FromSigMFVisitor::Field(size_t field_idx, size_t set_idx, flatbuffers::ElementaryType e_type, + bool is_vector, const flatbuffers::TypeTable *type_table, + const char *name, const uint8_t *val, json jj) { + *narrowest_json = jj; + last_field_name = name; + last_offset = flatbuffers::FieldIndexToOffset(static_cast(field_idx)); +} + + template + void FromSigMFVisitor::Named(T x, const char *name) { + try { + fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), T{}); + } catch (nlohmann::detail::out_of_range &e) { + } + } + +void FromSigMFVisitor::UType(uint8_t x, const char *name) { + Named(x, name); +} + + + void FromSigMFVisitor::Char(int8_t x, const char *name) { Named(x, name); } + + void FromSigMFVisitor::UChar(uint8_t x, const char *name) { Named(x, name); } + + void FromSigMFVisitor::Short(int16_t x, const char *name) { Named(x, name); } + + void FromSigMFVisitor::UShort(uint16_t x, const char *name) { Named(x, name); } + + void FromSigMFVisitor::Int(int32_t x, const char *name) { Named(x, name); } + + void FromSigMFVisitor::UInt(uint32_t x, const char *name) { Named(x, name); } + + void FromSigMFVisitor::Long(int64_t x) { + try { + fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), int64_t(0)); + } catch (nlohmann::detail::out_of_range &e) { + } + } + + void FromSigMFVisitor::ULong(uint64_t x) { + try { + fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), uint64_t(0)); + } catch (nlohmann::detail::out_of_range &e) { + } + } + + void FromSigMFVisitor::Float(float x) { + try { + fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), 0.f); + } catch (nlohmann::detail::out_of_range &e) { + } + } + + void FromSigMFVisitor::Double(double x) { + try { + fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), 0.0); + } catch (nlohmann::detail::out_of_range &e) { + } + } + + void FromSigMFVisitor::String(const struct flatbuffers::String *str) { + try { + auto strval = narrowest_json->at(p + last_field_name).get(); + last_fb_offset = flatbuffers::Offset(fbb.CreateString(strval).o); + } catch (nlohmann::detail::out_of_range &e) { + last_fb_offset.o = 0; + }; + } + + void FromSigMFVisitor::StartVector() { + throw std::runtime_error("the visitor was called on startvector... not implemented"); + } + + void FromSigMFVisitor::EndVector() { + throw std::runtime_error("the visitor was called on endvector... not implemented"); + } + + void FromSigMFVisitor::Element(size_t i, flatbuffers::ElementaryType /*type*/, + const flatbuffers::TypeTable * /*type_table*/, const uint8_t * /*val*/) {} + + +flatbuffers::Offset +json_vector_to_chararray(flatbuffers::FlatBufferBuilder &fbb, const json &jvec, flatbuffers::ElementaryType type) { + size_t dtype_size; + size_t num_elements = jvec.size(); + switch (type) { + case flatbuffers::ET_UTYPE: { + auto rvec = jvec.get >(); + return flatbuffers::Offset(fbb.CreateVector(rvec).o); + } + case flatbuffers::ET_BOOL: { + auto rvec = jvec.get >(); + return flatbuffers::Offset(fbb.CreateVector(rvec).o); + } + case flatbuffers::ET_CHAR: { + auto rvec = jvec.get >(); + return flatbuffers::Offset(fbb.CreateVector(rvec).o); + } + case flatbuffers::ET_UCHAR: { + auto rvec = jvec.get >(); + return flatbuffers::Offset(fbb.CreateVector(rvec).o); + } + case flatbuffers::ET_SHORT: + return json_vector_to_flatbuffer(fbb, jvec); + case flatbuffers::ET_USHORT: { + return json_vector_to_flatbuffer(fbb, jvec); + } + case flatbuffers::ET_INT: + return json_vector_to_flatbuffer(fbb, jvec); + case flatbuffers::ET_UINT: { + return json_vector_to_flatbuffer(fbb, jvec); + } + case flatbuffers::ET_LONG: + return json_vector_to_flatbuffer(fbb, jvec); + case flatbuffers::ET_ULONG: { + return json_vector_to_flatbuffer(fbb, jvec); + } + case flatbuffers::ET_FLOAT: { + return json_vector_to_flatbuffer(fbb, jvec); + } + case flatbuffers::ET_DOUBLE: { + return json_vector_to_flatbuffer(fbb, jvec); + } + default: + throw std::runtime_error("libsigmf cannot make a vector of this type yet"); + } +} + + +/** + * Iterate through a typetable-- I'll be honest here. This is kind of bullshit. We need to create all of + * the types like Strings, Lists, Vectors, and other flatbuffer types before we create our table. I'm not + * sure there's a better way to fill in the buffer than iterate over it twice-- i wonder how flatbuffers + * does this internally (look at Parsers). We're just going to construct shit in the fbb in the first loop + * and then fill in the fixed-width fields. I know this does the string creation correctly, but I don't + * know that lists(Vectors) or other tables/structs will work. The Vectors and Tables will probably require + * some recursion + * + * @param type_table the table to iterate over + * @param visitor the visitor responsible for creating objects and adding fields to its internal flatbufferbuilder + */ +void IterateType(const flatbuffers::TypeTable *type_table, FromSigMFVisitor *visitor, json original_json) { + const uint8_t *prev_val = nullptr; + const uint8_t val[8] = {}; + auto comosite_type_offsets = std::map >(); + // Generate the strings up front (because they must be created before our Table in the flatbuffer) + for (size_t i = 0; i < type_table->num_elems; i++) { + auto type_code = type_table->type_codes[i]; + auto type = static_cast(type_code.base_type); + auto is_vector = type_code.is_vector != 0; + auto ref_idx = type_code.sequence_ref; + const flatbuffers::TypeTable *ref = nullptr; + if (ref_idx >= 0) { ref = type_table->type_refs[ref_idx](); } + auto name = type_table->names ? type_table->names[i] : nullptr; + // The thing is, at least on this iteration we don't care about making real elementary types, just the + // ones that need an offset + if (type < flatbuffers::ET_STRING) { + if (is_vector) { + auto vector_offset = json_vector_to_chararray(visitor->fbb, original_json[name], type); + auto void_offset = flatbuffers::Offset(vector_offset.o); + comosite_type_offsets[i] = void_offset; + } + } else if (type == flatbuffers::ET_STRING) { + if (is_vector) { + std::vector > vecofstrings; + + for (const auto &item : original_json[visitor->p + name]) { + auto strval = item.get(); + auto last_fb_offset = flatbuffers::Offset(visitor->fbb.CreateString(strval).o); + + vecofstrings.emplace_back(last_fb_offset); + } + auto vecoffset = flatbuffers::Offset(visitor->fbb.CreateVector(vecofstrings).o); + comosite_type_offsets[i] = vecoffset; + } else { + if (!original_json[visitor->p + name].is_null()) { + visitor->Field(i, 0, type, is_vector, ref, name, val, original_json); + IterateValue(type, val, ref, prev_val, -1, visitor); + comosite_type_offsets[i] = visitor->last_fb_offset; + } + } + } else if (type == flatbuffers::ET_SEQUENCE) { + if (is_vector) { + std::vector > vecofstrings; + + for (const auto &item : original_json[visitor->p + name]) { + auto seq_typetable = type_table->type_refs[ref_idx](); + IterateType(seq_typetable, visitor, item); + auto vecelement = flatbuffers::Offset(visitor->_stop); + + vecofstrings.emplace_back(vecelement); + } + auto vecoffset = flatbuffers::Offset(visitor->fbb.CreateVector(vecofstrings).o); + comosite_type_offsets[i] = vecoffset; + } else { + auto seq_typetable = type_table->type_refs[ref_idx](); + + IterateType(seq_typetable, visitor, original_json[name]); + comosite_type_offsets[i] = flatbuffers::Offset(visitor->_stop); + } + } + + } + // Now make the table and fill in all of our types + size_t set_idx = 0; + visitor->StartSequence(); + for (size_t i = 0; i < type_table->num_elems; i++) { + auto type_code = type_table->type_codes[i]; + auto type = static_cast(type_code.base_type); + auto is_vector = type_code.is_vector != 0; + auto ref_idx = type_code.sequence_ref; + const flatbuffers::TypeTable *ref = nullptr; + if (ref_idx >= 0) { ref = type_table->type_refs[ref_idx](); } + auto name = type_table->names ? type_table->names[i] : nullptr; + if (is_vector) { + auto this_voffset = flatbuffers::FieldIndexToOffset(static_cast(i)); + visitor->fbb.AddOffset(this_voffset, comosite_type_offsets.at(i)); + } else if (type == flatbuffers::ET_STRING) { + try { + auto this_voffset = flatbuffers::FieldIndexToOffset(static_cast(i)); + visitor->fbb.AddOffset(this_voffset, comosite_type_offsets.at(i)); + } catch (std::out_of_range &e) { + // It's ok... this is just because a field wasn't present in json so we never had to deserialize + } + } else if (type == flatbuffers::ET_SEQUENCE) { + auto this_voffset = flatbuffers::FieldIndexToOffset(static_cast(i)); + visitor->fbb.AddOffset(this_voffset, comosite_type_offsets.at(i)); + } else { + visitor->Field(i, set_idx, type, is_vector, ref, name, val, original_json); + set_idx++; + IterateValue(type, val, ref, prev_val, -1, visitor); + } + } + visitor->EndSequence(); +} + +json +flatbuffer_field_to_json(const uint8_t *val, + flatbuffers::ElementaryType type, + const flatbuffers::TypeTable *tt, + const std::string &ns_prefix, + bool include_defaults) { + switch (type) { + case flatbuffers::ET_UTYPE: { + uint8_t tval = 0; + if (val) { + tval = *reinterpret_cast(val); + } + return json(tval); + break; + } + case flatbuffers::ET_BOOL: { + uint8_t tval = 0; + if (val) { + tval = json(*reinterpret_cast(val) != 0); + } + return json(tval); + break; + } + case flatbuffers::ET_CHAR: { + int8_t tval = 0; + if (val) { + tval = *reinterpret_cast(val); + } + return json(tval); + break; + } + case flatbuffers::ET_UCHAR: { + uint8_t tval = 0; + if (val) { + tval = *reinterpret_cast(val); + } + return json(tval); + break; + } + case flatbuffers::ET_SHORT: { + int16_t tval =0; + if (val) { + tval = *reinterpret_cast(val); + } + return json(tval); + break; + } + case flatbuffers::ET_USHORT: { + uint16_t tval = 0; + if (val) { + tval = *reinterpret_cast(val); + } + return json(tval); + break; + } + case flatbuffers::ET_INT: { + auto tval = 0; + if (val) { + tval = *reinterpret_cast(val); + } + return json(tval); + break; + } + case flatbuffers::ET_UINT: { + auto tval = 0UL; + if (val) { + tval = *reinterpret_cast(val); + } + return json(tval); + break; + } + case flatbuffers::ET_LONG: { + auto tval = 0LL; + if (val) { + tval = *reinterpret_cast(val); + } + return json(tval); + break; + } + case flatbuffers::ET_ULONG: { + auto tval = 0ULL; + if (val) { + tval = *reinterpret_cast(val); + } + return json(tval); + break; + } + case flatbuffers::ET_FLOAT: { + auto tval = 0.0; + if (val) { + tval = *reinterpret_cast(val); + } + return json(tval); + break; + } + case flatbuffers::ET_DOUBLE: { + auto tval = 0.0; + if (val) { + tval = *reinterpret_cast(val); + } + return json(tval); + break; + } + case flatbuffers::ET_STRING: { + std::string tval{}; + if (val) { + val += flatbuffers::ReadScalar(val); + tval = reinterpret_cast(val)->c_str(); + } + return json(tval); + break; + } + case flatbuffers::ET_SEQUENCE: { + switch (tt->st) { + case flatbuffers::ST_TABLE: + val += flatbuffers::ReadScalar(val); + return FlatBufferToJson(val, tt, ns_prefix, include_defaults); + // Have not implemented structs, unions, or enums which are all supported by flatbuffers schema + // and would be great to have, but dont make as much sense in json-land. They're on the todo list... + // here is what flatbuffers does internally (see IterateValue in minireflect.h) + default: + throw std::runtime_error( + "libsigmf has not implemented flatbuffer struct, union, or enum to json yet"); +// case flatbuffers::ST_STRUCT: IterateObject(val, type_table, visitor); break; +// case flatbuffers::ST_UNION: { +// val += flatbuffers::ReadScalar(val); +// FLATBUFFERS_ASSERT(prev_val); +// auto union_type = *prev_val; // Always a uint8_t. +// if (vector_index >= 0) { +// auto type_vec = reinterpret_cast *>(prev_val); +// union_type = type_vec->Get(static_cast(vector_index)); +// } +// auto type_code_idx = +// flatbuffers::LookupEnum(union_type, type_table->values, type_table->num_elems); +// if (type_code_idx >= 0 && +// type_code_idx < static_cast(type_table->num_elems)) { +// auto type_code = type_table->type_codes[type_code_idx]; +// switch (type_code.base_type) { +// case flatbuffers::ET_SEQUENCE: { +// auto ref = type_table->type_refs[type_code.sequence_ref](); +// IterateObject(val, ref, visitor); +// break; +// } +// case flatbuffers::ET_STRING: +// visitor->String(reinterpret_cast(val)); +// break; +// default: visitor->Unknown(val); +// } +// } else { +// visitor->Unknown(val); +// } +// break; +// } +// case flatbuffers::ST_ENUM: FLATBUFFERS_ASSERT(false); break; + + } + } + } +} + +json +FlatBufferToJson(const uint8_t *buffer_root, const flatbuffers::TypeTable *typetable, const std::string &ns_prefix, + bool include_defaults) { + json json_object; + const auto obj = reinterpret_cast(buffer_root); + for (size_t i = 0; i < typetable->num_elems; i++) { + // Gather all of the underlying info about this element in table + auto type_code = typetable->type_codes[i]; + auto type = static_cast(type_code.base_type); + auto is_vector = type_code.is_vector != 0; + auto ref_idx = type_code.sequence_ref; + const flatbuffers::TypeTable *ref = nullptr; + if (ref_idx >= 0) { ref = typetable->type_refs[ref_idx](); } + auto name = typetable->names ? typetable->names[i] : nullptr; + const uint8_t *val = nullptr; + + // Fetch the actual value of this field now and stick it inside our json object + if (typetable->st == flatbuffers::ST_TABLE) { + auto field_offset = flatbuffers::FieldIndexToOffset(static_cast(i)); + val = obj->GetAddressOf(field_offset); + } else { + // when does this get triggered? Probably when someone tries to pass a non-flattbuffers object + // to us. Let's throw an exception so it'll be nice and obvious with a good test-case when this + // happens. + throw std::runtime_error( + "libsigmf::FlatBufferToJson was called with a non-table type. Please file an issue with your fbs schema and input data."); + // Here is what flatbuffers does internally: val = obj + typetable->values[i]; + } + const flatbuffers::TypeTable *ttptr = nullptr; + if (val) { + if (ref_idx >= 0) { + ttptr = typetable->type_refs[ref_idx](); + } + if (is_vector) { + val += flatbuffers::ReadScalar(val); + auto vec = reinterpret_cast *>(val); + auto elem_ptr = vec->Data(); + for (size_t j = 0; j < vec->size(); j++) { + json_object[ns_prefix + name].push_back( + flatbuffer_field_to_json(elem_ptr, type, ttptr, ns_prefix, include_defaults)); + elem_ptr += InlineSize(type, ref); + } + } else { + json lval = flatbuffer_field_to_json(val, type, ttptr, ns_prefix, include_defaults); + json_object[ns_prefix + name] = lval; + } + } else { + // There is no value in this field. This is OK except maybe we want to figure out how to enforce + // required fields. Although maybe flatbuffers schema language and the flatbuffersbuilders have + // already enforced such things. + } + if (!is_vector && type < flatbuffers::ET_SEQUENCE && include_defaults) { + json lval = flatbuffer_field_to_json(val, type, ttptr, ns_prefix, include_defaults); + json_object[ns_prefix + name] = lval; + } + } + return json_object; +} \ No newline at end of file diff --git a/src/lib/json_wrap.cpp b/src/lib/json_wrap.cpp new file mode 100644 index 0000000..c24b74b --- /dev/null +++ b/src/lib/json_wrap.cpp @@ -0,0 +1,3 @@ + +#include "libsigmf/json_wrap.h" +#include \ No newline at end of file From 57283cf5e00ea35405b27e71809c76100a529f47 Mon Sep 17 00:00:00 2001 From: Nathan West Date: Sat, 11 Apr 2020 21:53:24 -0400 Subject: [PATCH 3/4] minimize public interface and create forward definitions --- examples/example_reading_sigmf_file.cpp | 2 +- .../libsigmf/flatbuffers_json_visitor.h | 18 -- src/include/libsigmf/sigmf.h | 89 ++++---- src/include/libsigmf/sigmf_forward.h | 50 +++++ src/include/libsigmf/sigmf_helpers.h | 12 +- src/include/libsigmf/variadic_data_class.h | 1 + .../libsigmf/variadic_data_class_forward.h | 27 +++ src/lib/flatbuffers_json_visitor.cpp | 209 +++++++++--------- 8 files changed, 239 insertions(+), 169 deletions(-) create mode 100644 src/include/libsigmf/sigmf_forward.h create mode 100644 src/include/libsigmf/variadic_data_class_forward.h diff --git a/examples/example_reading_sigmf_file.cpp b/examples/example_reading_sigmf_file.cpp index a3fef25..c45c0c7 100644 --- a/examples/example_reading_sigmf_file.cpp +++ b/examples/example_reading_sigmf_file.cpp @@ -27,5 +27,5 @@ int main(int argc, char* argv[]) { auto record = sigmf::metadata_file_to_json(meta_fstream); std::cout << "The record we read is: \n" << - record->to_json().dump(2) << std::endl; + to_json(record).dump(2) << std::endl; } \ No newline at end of file diff --git a/src/include/libsigmf/flatbuffers_json_visitor.h b/src/include/libsigmf/flatbuffers_json_visitor.h index 8f325bb..6920872 100644 --- a/src/include/libsigmf/flatbuffers_json_visitor.h +++ b/src/include/libsigmf/flatbuffers_json_visitor.h @@ -19,7 +19,6 @@ #include "json_wrap.h" #include -#include /* @@ -83,9 +82,6 @@ struct FromSigMFVisitor : public flatbuffers::IterationVisitor { const flatbuffers::TypeTable * /*type_table*/, const uint8_t * /*val*/) override; }; -flatbuffers::Offset -json_vector_to_chararray(flatbuffers::FlatBufferBuilder &fbb, const json &jvec, flatbuffers::ElementaryType type); - /** * Iterate through a typetable-- I'll be honest here. This is kind of bullshit. We need to create all of * the types like Strings, Lists, Vectors, and other flatbuffer types before we create our table. I'm not @@ -100,19 +96,6 @@ json_vector_to_chararray(flatbuffers::FlatBufferBuilder &fbb, const json &jvec, */ void IterateType(const flatbuffers::TypeTable *type_table, FromSigMFVisitor *visitor, json original_json); -// forward declare so we can create objects as fields -json -FlatBufferToJson(const uint8_t *buffer_root, const flatbuffers::TypeTable *typetable, const std::string &ns_prefix, - bool include_defaults = false); - -json -flatbuffer_field_to_json(const uint8_t *val, - flatbuffers::ElementaryType type, - const flatbuffers::TypeTable *tt = nullptr, - const std::string &ns_prefix = "", - bool include_defaults = false); - - /** * A function to iterate through a flatbuffer that is described by the type and build up a json object to return. * The flatbuffer should have already been Finished, and the result of FlatBufferBuilder::Finish() -> GetRoot should @@ -128,5 +111,4 @@ json FlatBufferToJson(const uint8_t *buffer_root, const flatbuffers::TypeTable *typetable, const std::string &ns_prefix, bool include_defaults); - #endif //LIBSIGMF_FLATBUFFERS_TO_JSON_VISITOR_H diff --git a/src/include/libsigmf/sigmf.h b/src/include/libsigmf/sigmf.h index f5fa24c..fb20e57 100644 --- a/src/include/libsigmf/sigmf.h +++ b/src/include/libsigmf/sigmf.h @@ -20,72 +20,71 @@ #include "global.h" #include "capture.h" #include "annotation.h" +#include "json_wrap.h" +#include +#include "sigmf_forward.h" #include "sigmf_core_generated.h" #include namespace sigmf { - template - class SigMFVector : public std::vector { - public: - T &create_new() { - T new_element; - this->emplace_back(new_element); - return this->back(); - } - }; + /* + * This makes conversion between json types and SigMF types work out of the box + */ + + template + json to_json(const SigMF t) { + json j; + j["global"] = t.global.to_json(); + j["captures"] = t.captures; + j["annotations"] = t.annotations; + return j; + } template - struct SigMF { - GlobalType global; - SigMFVector captures; - SigMFVector annotations; + void to_json(json &j, const SigMF t) { + j["global"] = t.global.to_json(); + j["captures"] = t.captures; + j["annotations"] = t.annotations; + } - /** - * Export the record to a JSON object - */ - json to_json() const { - json j; - j["global"] = global.to_json(); - j["captures"] = captures; - j["annotations"] = annotations; - return j; + template + SigMF from_json(const json &j) { + SigMF t; + t.global.from_json(j["global"]); + t.captures.clear(); + t.annotations.clear(); + for (auto &element : j["annotations"]) { + AnnotationType a; + a.from_json(element); + t.annotations.emplace_back(a); + } + for (auto &element : j["captures"]) { + CaptureType c; + c.from_json(element); + t.captures.emplace_back(c); } + return t; + } - /** - * Write over the fields with a new record from a JSON object - */ - void from_json(const json &j) { - global.from_json(j["global"]); - captures.clear(); - annotations.clear(); + template + void from_json(const json &j, SigMF &t) { + t.global.from_json(j["global"]); + t.captures.clear(); + t.annotations.clear(); for (auto &element : j["annotations"]) { AnnotationType a; a.from_json(element); - annotations.emplace_back(a); + t.annotations.emplace_back(a); } for (auto &element : j["captures"]) { CaptureType c; c.from_json(element); - captures.emplace_back(c); + t.captures.emplace_back(c); } } - }; - - /* - * This makes conversion between json types and SigMF types work out of the box - */ - - template - void to_json(json &j, const SigMF t) { - j = t.to_json(); } - template - void from_json(const json &j, SigMF &t) { - t.from_json(j); - } -} #endif //LIBSIGMF_SIGMF_H diff --git a/src/include/libsigmf/sigmf_forward.h b/src/include/libsigmf/sigmf_forward.h new file mode 100644 index 0000000..9b61da1 --- /dev/null +++ b/src/include/libsigmf/sigmf_forward.h @@ -0,0 +1,50 @@ +/* + * Copyright 2019 DeepSig Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBSIGMF_SIGMF_FORWARD_H +#define LIBSIGMF_SIGMF_FORWARD_H + +namespace sigmf { + + template + class Global; + + template + class Captures; + + template + class Annotations; + + template + class SigMFVector : public std::vector { + public: + T &create_new() { + T new_element; + this->emplace_back(new_element); + return this->back(); + } + }; + + template + struct SigMF { + GlobalType global; + SigMFVector captures; + SigMFVector annotations; + }; + +} + +#endif // LIBSIGMF_SIGMF_FORWARD_H \ No newline at end of file diff --git a/src/include/libsigmf/sigmf_helpers.h b/src/include/libsigmf/sigmf_helpers.h index d1fa585..88ddcd9 100644 --- a/src/include/libsigmf/sigmf_helpers.h +++ b/src/include/libsigmf/sigmf_helpers.h @@ -171,17 +171,17 @@ namespace sigmf { * @return Pointer to SigMF object * */ - static std::unique_ptr, + static sigmf::SigMF, sigmf::Capture, - sigmf::Annotation>> metadata_file_to_json(const std::ifstream &meta_file) { + sigmf::Annotation> metadata_file_to_json(const std::ifstream &meta_file) { std::ostringstream meta_buffer; meta_buffer << meta_file.rdbuf(); - auto sigmf_md = std::make_unique, + auto sigmf_md = sigmf::SigMF, sigmf::Capture, - sigmf::Annotation>>(); - - *sigmf_md = json::parse(meta_buffer.str()); + sigmf::Annotation>(); + + from_json(json::parse(meta_buffer.str()), sigmf_md); return sigmf_md; } diff --git a/src/include/libsigmf/variadic_data_class.h b/src/include/libsigmf/variadic_data_class.h index 20ccd18..14c05a0 100644 --- a/src/include/libsigmf/variadic_data_class.h +++ b/src/include/libsigmf/variadic_data_class.h @@ -17,6 +17,7 @@ #ifndef LIBSIGMF_VARIADICDATACLASS_H #define LIBSIGMF_VARIADICDATACLASS_H +#include "variadic_data_class_forward.h" #include "flatbuffers_json_visitor.h" #include "json_wrap.h" #include diff --git a/src/include/libsigmf/variadic_data_class_forward.h b/src/include/libsigmf/variadic_data_class_forward.h new file mode 100644 index 0000000..f0d1312 --- /dev/null +++ b/src/include/libsigmf/variadic_data_class_forward.h @@ -0,0 +1,27 @@ +/* + * Copyright 2019 DeepSig Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBSIGMF_VARIADICDATACLASS_FORWARD_H +#define LIBSIGMF_VARIADICDATACLASS_FORWARD_H + +namespace sigmf { + + template + class VariadicDataClass; + +} + +#endif // LIBSIGMF_VARIADICDATACLASS_FORWARD_H \ No newline at end of file diff --git a/src/lib/flatbuffers_json_visitor.cpp b/src/lib/flatbuffers_json_visitor.cpp index 18db45a..49c9c56 100644 --- a/src/lib/flatbuffers_json_visitor.cpp +++ b/src/lib/flatbuffers_json_visitor.cpp @@ -16,102 +16,6 @@ flatbuffers::Offset json_vector_to_flatbuffer(flatbuffers::FlatBufferBuild return flatbuffers::Offset(fbb.CreateVector(tmpvec).o); } -FromSigMFVisitor::FromSigMFVisitor(std::string namespace_prefix, const json &j) - : p(namespace_prefix), last_offset(0), _start(0), _stop(0) { - fbb = flatbuffers::FlatBufferBuilder(); - narrowest_json = std::make_unique(); - *narrowest_json = j; -} - -void FromSigMFVisitor::StartSequence() { - _start = fbb.StartTable(); -} - -void FromSigMFVisitor::EndSequence() { - _stop = fbb.EndTable(_start); -} - -void FromSigMFVisitor::Field(size_t field_idx, size_t set_idx, flatbuffers::ElementaryType e_type, - bool is_vector, const flatbuffers::TypeTable *type_table, - const char *name, const uint8_t *val, json jj) { - *narrowest_json = jj; - last_field_name = name; - last_offset = flatbuffers::FieldIndexToOffset(static_cast(field_idx)); -} - - template - void FromSigMFVisitor::Named(T x, const char *name) { - try { - fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), T{}); - } catch (nlohmann::detail::out_of_range &e) { - } - } - -void FromSigMFVisitor::UType(uint8_t x, const char *name) { - Named(x, name); -} - - - void FromSigMFVisitor::Char(int8_t x, const char *name) { Named(x, name); } - - void FromSigMFVisitor::UChar(uint8_t x, const char *name) { Named(x, name); } - - void FromSigMFVisitor::Short(int16_t x, const char *name) { Named(x, name); } - - void FromSigMFVisitor::UShort(uint16_t x, const char *name) { Named(x, name); } - - void FromSigMFVisitor::Int(int32_t x, const char *name) { Named(x, name); } - - void FromSigMFVisitor::UInt(uint32_t x, const char *name) { Named(x, name); } - - void FromSigMFVisitor::Long(int64_t x) { - try { - fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), int64_t(0)); - } catch (nlohmann::detail::out_of_range &e) { - } - } - - void FromSigMFVisitor::ULong(uint64_t x) { - try { - fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), uint64_t(0)); - } catch (nlohmann::detail::out_of_range &e) { - } - } - - void FromSigMFVisitor::Float(float x) { - try { - fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), 0.f); - } catch (nlohmann::detail::out_of_range &e) { - } - } - - void FromSigMFVisitor::Double(double x) { - try { - fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), 0.0); - } catch (nlohmann::detail::out_of_range &e) { - } - } - - void FromSigMFVisitor::String(const struct flatbuffers::String *str) { - try { - auto strval = narrowest_json->at(p + last_field_name).get(); - last_fb_offset = flatbuffers::Offset(fbb.CreateString(strval).o); - } catch (nlohmann::detail::out_of_range &e) { - last_fb_offset.o = 0; - }; - } - - void FromSigMFVisitor::StartVector() { - throw std::runtime_error("the visitor was called on startvector... not implemented"); - } - - void FromSigMFVisitor::EndVector() { - throw std::runtime_error("the visitor was called on endvector... not implemented"); - } - - void FromSigMFVisitor::Element(size_t i, flatbuffers::ElementaryType /*type*/, - const flatbuffers::TypeTable * /*type_table*/, const uint8_t * /*val*/) {} - flatbuffers::Offset json_vector_to_chararray(flatbuffers::FlatBufferBuilder &fbb, const json &jvec, flatbuffers::ElementaryType type) { @@ -160,7 +64,6 @@ json_vector_to_chararray(flatbuffers::FlatBufferBuilder &fbb, const json &jvec, } } - /** * Iterate through a typetable-- I'll be honest here. This is kind of bullshit. We need to create all of * the types like Strings, Lists, Vectors, and other flatbuffer types before we create our table. I'm not @@ -267,7 +170,7 @@ void IterateType(const flatbuffers::TypeTable *type_table, FromSigMFVisitor *vis } visitor->EndSequence(); } - + json flatbuffer_field_to_json(const uint8_t *val, flatbuffers::ElementaryType type, @@ -428,6 +331,17 @@ flatbuffer_field_to_json(const uint8_t *val, } } +/** + * A function to iterate through a flatbuffer that is described by the type and build up a json object to return. + * The flatbuffer should have already been Finished, and the result of FlatBufferBuilder::Finish() -> GetRoot should + * be the buffer_root. See variadic_data_class where we do the following: + * auto bfrptr = fbb.GetBufferPointer(); + * auto rtptr = flatbuffers::GetRoot(bfrptr); + * + * The gist of this is to get information about each field inside of a flatbuffers Table, then for each field, check + * if there is a value. If so, serialize it to json and shove its value inside the json_object we want to return + * using the name of the field as the key. + */ json FlatBufferToJson(const uint8_t *buffer_root, const flatbuffers::TypeTable *typetable, const std::string &ns_prefix, bool include_defaults) { @@ -485,4 +399,101 @@ FlatBufferToJson(const uint8_t *buffer_root, const flatbuffers::TypeTable *typet } } return json_object; -} \ No newline at end of file +} + + +FromSigMFVisitor::FromSigMFVisitor(std::string namespace_prefix, const json &j) + : p(namespace_prefix), last_offset(0), _start(0), _stop(0) { + fbb = flatbuffers::FlatBufferBuilder(); + narrowest_json = std::make_unique(); + *narrowest_json = j; +} + +void FromSigMFVisitor::StartSequence() { + _start = fbb.StartTable(); +} + +void FromSigMFVisitor::EndSequence() { + _stop = fbb.EndTable(_start); +} + +void FromSigMFVisitor::Field(size_t field_idx, size_t set_idx, flatbuffers::ElementaryType e_type, + bool is_vector, const flatbuffers::TypeTable *type_table, + const char *name, const uint8_t *val, json jj) { + *narrowest_json = jj; + last_field_name = name; + last_offset = flatbuffers::FieldIndexToOffset(static_cast(field_idx)); +} + + template + void FromSigMFVisitor::Named(T x, const char *name) { + try { + fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), T{}); + } catch (nlohmann::detail::out_of_range &e) { + } + } + +void FromSigMFVisitor::UType(uint8_t x, const char *name) { + Named(x, name); +} + + + void FromSigMFVisitor::Char(int8_t x, const char *name) { Named(x, name); } + + void FromSigMFVisitor::UChar(uint8_t x, const char *name) { Named(x, name); } + + void FromSigMFVisitor::Short(int16_t x, const char *name) { Named(x, name); } + + void FromSigMFVisitor::UShort(uint16_t x, const char *name) { Named(x, name); } + + void FromSigMFVisitor::Int(int32_t x, const char *name) { Named(x, name); } + + void FromSigMFVisitor::UInt(uint32_t x, const char *name) { Named(x, name); } + + void FromSigMFVisitor::Long(int64_t x) { + try { + fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), int64_t(0)); + } catch (nlohmann::detail::out_of_range &e) { + } + } + + void FromSigMFVisitor::ULong(uint64_t x) { + try { + fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), uint64_t(0)); + } catch (nlohmann::detail::out_of_range &e) { + } + } + + void FromSigMFVisitor::Float(float x) { + try { + fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), 0.f); + } catch (nlohmann::detail::out_of_range &e) { + } + } + + void FromSigMFVisitor::Double(double x) { + try { + fbb.AddElement(last_offset, narrowest_json->at(p + last_field_name).get(), 0.0); + } catch (nlohmann::detail::out_of_range &e) { + } + } + + void FromSigMFVisitor::String(const struct flatbuffers::String *str) { + try { + auto strval = narrowest_json->at(p + last_field_name).get(); + last_fb_offset = flatbuffers::Offset(fbb.CreateString(strval).o); + } catch (nlohmann::detail::out_of_range &e) { + last_fb_offset.o = 0; + }; + } + + void FromSigMFVisitor::StartVector() { + throw std::runtime_error("the visitor was called on startvector... not implemented"); + } + + void FromSigMFVisitor::EndVector() { + throw std::runtime_error("the visitor was called on endvector... not implemented"); + } + + void FromSigMFVisitor::Element(size_t i, flatbuffers::ElementaryType /*type*/, + const flatbuffers::TypeTable * /*type_table*/, const uint8_t * /*val*/) {} From 1d28bf69b90835c2e30826379eb98acb56a17fde Mon Sep 17 00:00:00 2001 From: Nathan West Date: Sat, 11 Apr 2020 22:00:49 -0400 Subject: [PATCH 4/4] update nlohmann json to v3.7.3 --- external/json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/json b/external/json index 68ec3eb..e7b3b40 160000 --- a/external/json +++ b/external/json @@ -1 +1 @@ -Subproject commit 68ec3eb8d633770fabb2029c10511bd4664577dc +Subproject commit e7b3b40b5a95bc74b9a7f662830a27c49ffc01b4