diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt index 2691d9d33a0c6..17320348d9272 100644 --- a/Framework/Core/CMakeLists.txt +++ b/Framework/Core/CMakeLists.txt @@ -140,6 +140,7 @@ o2_add_library(Framework src/Task.cxx src/Array2D.cxx src/Variant.cxx + src/VariantJSONHelpers.cxx src/VariantPropertyTreeHelpers.cxx src/WorkflowCustomizationHelpers.cxx src/WorkflowHelpers.cxx diff --git a/Framework/Core/include/Framework/VariantJSONHelpers.h b/Framework/Core/include/Framework/VariantJSONHelpers.h index eab78d547ca13..811e6f13d4985 100644 --- a/Framework/Core/include/Framework/VariantJSONHelpers.h +++ b/Framework/Core/include/Framework/VariantJSONHelpers.h @@ -19,439 +19,15 @@ #include #include -#include -#include -#include +#include namespace o2::framework { -namespace -{ -template -struct VariantReader : public rapidjson::BaseReaderHandler, VariantReader> { - using Ch = rapidjson::UTF8<>::Ch; - using SizeType = rapidjson::SizeType; - - enum struct State { - IN_START, - IN_STOP, - IN_DATA, - IN_KEY, - IN_ARRAY, - IN_ROW, - IN_ERROR - }; - - VariantReader() - : states{}, - rows{0}, - cols{0} - { - debug << "Start" << std::endl; - states.push(State::IN_START); - } - - bool Null() - { - debug << "Null value encountered" << std::endl; - return true; - } - - bool Int(int i) - { - debug << "Int(" << i << ")" << std::endl; - if (states.top() == State::IN_ERROR) { - debug << "In ERROR state" << std::endl; - return false; - } - if constexpr (!std::is_same_v>) { - states.push(State::IN_ERROR); - return true; - } else { - if (states.top() == State::IN_ARRAY || states.top() == State::IN_ROW) { - debug << "added to array" << std::endl; - accumulatedData.push_back(i); - return true; - } - } - states.push(State::IN_ERROR); - return true; - } - - bool Uint(unsigned i) - { - debug << "Uint -> Int" << std::endl; - return Int(static_cast(i)); - } - - bool Int64(int64_t i) - { - debug << "Int64 -> Int" << std::endl; - return Int(static_cast(i)); - } - - bool Uint64(uint64_t i) - { - debug << "Uint64 -> Int" << std::endl; - return Int(static_cast(i)); - } - - bool Double(double d) - { - debug << "Double(" << d << ")" << std::endl; - if (states.top() == State::IN_ERROR) { - debug << "In ERROR state" << std::endl; - return false; - } - if constexpr (!(std::is_same_v> || std::is_same_v>)) { - states.push(State::IN_ERROR); - return true; - } - if (states.top() == State::IN_ARRAY || states.top() == State::IN_ROW) { - if constexpr (std::is_same_v>) { - debug << "added to array as double" << std::endl; - accumulatedData.push_back(d); - return true; - } else if constexpr (std::is_same_v>) { - debug << "added to array as float" << std::endl; - accumulatedData.push_back(static_cast(d)); - return true; - } - } - states.push(State::IN_ERROR); - return true; - } - - bool Bool(bool b) - { - debug << "Bool(" << b << ")" << std::endl; - if (states.top() == State::IN_ERROR) { - debug << "In ERROR state" << std::endl; - return false; - } - if constexpr (!std::is_same_v>) { - states.push(State::IN_ERROR); - return false; - } else { - if (states.top() == State::IN_ARRAY) { - debug << "added to array" << std::endl; - accumulatedData.push_back(b); - return true; - } - states.push(State::IN_ERROR); - return true; - } - } - - bool String(const Ch* str, SizeType, bool) - { - debug << "String(" << str << ")" << std::endl; - if (states.top() == State::IN_ERROR) { - debug << "In ERROR state" << std::endl; - return false; - } - if constexpr (!(V == VariantType::ArrayString || isLabeledArray())) { - states.push(State::IN_ERROR); - return true; - } else { - if (states.top() == State::IN_ARRAY || states.top() == State::IN_ROW) { - debug << "added to array" << std::endl; - if constexpr (isLabeledArray()) { - if (currentKey == labels_rows_str) { - labels_rows.push_back(str); - return true; - } - if (currentKey == labels_cols_str) { - labels_cols.push_back(str); - return true; - } - } - if (currentKey == "values") { - if constexpr (std::is_same_v>) { - accumulatedData.push_back(str); - } else { - states.push(State::IN_ERROR); - } - return true; - } - return true; - } - states.push(State::IN_ERROR); - return true; - } - } - - bool StartObject() - { - debug << "StartObject()" << std::endl; - if (states.top() == State::IN_ERROR) { - debug << "In ERROR state" << std::endl; - return false; - } - if (states.top() == State::IN_START) { - states.push(State::IN_DATA); - return true; - } - states.push(State::IN_ERROR); - return true; - } - - bool Key(const Ch* str, SizeType, bool) - { - debug << "Key(" << str << ")" << std::endl; - if (states.top() == State::IN_ERROR) { - debug << "In ERROR state" << std::endl; - currentKey = str; - return false; - } - if (states.top() == State::IN_DATA) { - // no previous keys - states.push(State::IN_KEY); - currentKey = str; - return true; - } - if (states.top() == State::IN_KEY) { - currentKey = str; - if constexpr (!isLabeledArray()) { - debug << "extra keys in a single-key variant" << std::endl; - states.push(State::IN_ERROR); - return true; - } - return true; - } - currentKey = str; - states.push(State::IN_ERROR); - return true; - } - - bool EndObject(SizeType) - { - debug << "EndObject()" << std::endl; - if (states.top() == State::IN_ERROR) { - debug << "In ERROR state" << std::endl; - return false; - } - if (states.top() == State::IN_KEY) { - if constexpr (isArray()) { - debug << "creating 1d-array variant" << std::endl; - result = Variant(accumulatedData); - } else if constexpr (isArray2D()) { - debug << "creating 2d-array variant" << std::endl; - assert(accumulatedData.size() == rows * cols); - result = Variant(Array2D{accumulatedData, rows, cols}); - } else if constexpr (isLabeledArray()) { - debug << "creating labeled array variant" << std::endl; - assert(accumulatedData.size() == rows * cols); - if (labels_rows.empty() == false) { - assert(labels_rows.size() == rows); - } - if (labels_cols.empty() == false) { - assert(labels_cols.size() == cols); - } - result = Variant(LabeledArray{Array2D{accumulatedData, rows, cols}, labels_rows, labels_cols}); - } - states.push(State::IN_STOP); - return true; - } - states.push(State::IN_ERROR); - return true; - } - - bool StartArray() - { - debug << "StartArray()" << std::endl; - if (states.top() == State::IN_ERROR) { - debug << "In ERROR state" << std::endl; - return false; - } - if (states.top() == State::IN_KEY) { - states.push(State::IN_ARRAY); - return true; - } else if (states.top() == State::IN_ARRAY) { - if constexpr (isArray2D() || isLabeledArray()) { - states.push(State::IN_ROW); - return true; - } - } - states.push(State::IN_ERROR); - return true; - } - - bool EndArray(SizeType elementCount) - { - debug << "EndArray()" << std::endl; - if (states.top() == State::IN_ERROR) { - debug << "In ERROR state" << std::endl; - return false; - } - if (states.top() == State::IN_ARRAY) { - // finish up array - states.pop(); - if constexpr (isArray2D() || isLabeledArray()) { - rows = elementCount; - } - return true; - } else if (states.top() == State::IN_ROW) { - // finish up row - states.pop(); - if constexpr (isArray2D() || isLabeledArray()) { - cols = elementCount; - } - return true; - } - states.push(State::IN_ERROR); - return true; - } - - std::stack states; - std::ostringstream debug; - - uint32_t rows; - uint32_t cols; - std::string currentKey; - std::vector> accumulatedData; - std::vector labels_rows; - std::vector labels_cols; - Variant result; -}; - -template -void writeVariant(std::ostream& o, Variant const& v) -{ - if constexpr (isArray() || isArray2D() || isLabeledArray()) { - using type = variant_array_element_type_t; - rapidjson::OStreamWrapper osw(o); - rapidjson::Writer w(osw); - - auto writeArray = [&](auto* values, size_t size) { - using T = std::remove_pointer_t; - w.StartArray(); - for (auto i = 0u; i < size; ++i) { - if constexpr (std::is_same_v) { - w.Int(values[i]); - } else if constexpr (std::is_same_v || std::is_same_v) { - w.Double(values[i]); - } else if constexpr (std::is_same_v) { - w.Bool(values[i]); - } else if constexpr (std::is_same_v) { - w.String(values[i].c_str()); - } - } - w.EndArray(); - }; - - auto writeVector = [&](auto&& vector) { - return writeArray(vector.data(), vector.size()); - }; - - auto writeArray2D = [&](auto&& array2d) { - using T = typename std::decay_t::element_t; - w.StartArray(); - for (auto i = 0u; i < array2d.rows; ++i) { - w.StartArray(); - for (auto j = 0u; j < array2d.cols; ++j) { - if constexpr (std::is_same_v) { - w.Int(array2d(i, j)); - } else if constexpr (std::is_same_v || std::is_same_v) { - w.Double(array2d(i, j)); - } else if constexpr (std::is_same_v) { - w.String(array2d(i, j).c_str()); - } - } - w.EndArray(); - } - w.EndArray(); - }; - - auto writeLabeledArray = [&](auto&& array) { - w.Key(labels_rows_str); - writeVector(array.getLabelsRows()); - w.Key(labels_cols_str); - writeVector(array.getLabelsCols()); - w.Key("values"); - writeArray2D(array.getData()); - }; - - w.StartObject(); - if constexpr (isArray()) { - w.Key("values"); - writeArray(v.get(), v.size()); - } else if constexpr (isArray2D()) { - w.Key("values"); - writeArray2D(v.get>()); - } else if constexpr (isLabeledArray()) { - writeLabeledArray(v.get>()); - } else if constexpr (V == VariantType::Dict) { - // nothing to do for dicts - } - w.EndObject(); - } -} -} // namespace - struct VariantJSONHelpers { template - static Variant read(std::istream& s) - { - rapidjson::Reader reader; - rapidjson::IStreamWrapper isw(s); - VariantReader vreader; - bool ok = reader.Parse(isw, vreader); - - if (ok == false) { - std::stringstream error; - error << "Cannot parse serialized Variant, error: " << rapidjson::GetParseError_En(reader.GetParseErrorCode()) << " at offset: " << reader.GetErrorOffset(); - throw std::runtime_error(error.str()); - } - return vreader.result; - } + static Variant read(std::istream& s); - static void write(std::ostream& o, Variant const& v) - { - switch (v.type()) { - case VariantType::ArrayInt: - writeVariant(o, v); - break; - case VariantType::ArrayFloat: - writeVariant(o, v); - break; - case VariantType::ArrayDouble: - writeVariant(o, v); - break; - case VariantType::ArrayBool: - throw std::runtime_error("Bool vectors not implemented yet"); - // writeVariant(o, v); - break; - case VariantType::ArrayString: - writeVariant(o, v); - break; - case VariantType::Array2DInt: - writeVariant(o, v); - break; - case VariantType::Array2DFloat: - writeVariant(o, v); - break; - case VariantType::Array2DDouble: - writeVariant(o, v); - break; - case VariantType::LabeledArrayInt: - writeVariant(o, v); - break; - case VariantType::LabeledArrayFloat: - writeVariant(o, v); - break; - case VariantType::LabeledArrayDouble: - writeVariant(o, v); - break; - case VariantType::LabeledArrayString: - writeVariant(o, v); - break; - case VariantType::Dict: - writeVariant(o, v); - default: - break; - } - } + static void write(std::ostream& o, Variant const& v); }; } // namespace o2::framework diff --git a/Framework/Core/src/VariantJSONHelpers.cxx b/Framework/Core/src/VariantJSONHelpers.cxx new file mode 100644 index 0000000000000..fbb5abb331867 --- /dev/null +++ b/Framework/Core/src/VariantJSONHelpers.cxx @@ -0,0 +1,464 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +#include "Framework/VariantJSONHelpers.h" +#include "Framework/Variant.h" + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace o2::framework +{ +namespace +{ +template +struct VariantReader : public rapidjson::BaseReaderHandler, VariantReader> { + using Ch = rapidjson::UTF8<>::Ch; + using SizeType = rapidjson::SizeType; + + enum struct State { + IN_START, + IN_STOP, + IN_DATA, + IN_KEY, + IN_ARRAY, + IN_ROW, + IN_ERROR + }; + + VariantReader() + : states{}, + rows{0}, + cols{0} + { + debug << "Start" << std::endl; + states.push(State::IN_START); + } + + bool Null() + { + debug << "Null value encountered" << std::endl; + return true; + } + + bool Int(int i) + { + debug << "Int(" << i << ")" << std::endl; + if (states.top() == State::IN_ERROR) { + debug << "In ERROR state" << std::endl; + return false; + } + if constexpr (!std::is_same_v>) { + states.push(State::IN_ERROR); + return true; + } else { + if (states.top() == State::IN_ARRAY || states.top() == State::IN_ROW) { + debug << "added to array" << std::endl; + accumulatedData.push_back(i); + return true; + } + } + states.push(State::IN_ERROR); + return true; + } + + bool Uint(unsigned i) + { + debug << "Uint -> Int" << std::endl; + return Int(static_cast(i)); + } + + bool Int64(int64_t i) + { + debug << "Int64 -> Int" << std::endl; + return Int(static_cast(i)); + } + + bool Uint64(uint64_t i) + { + debug << "Uint64 -> Int" << std::endl; + return Int(static_cast(i)); + } + + bool Double(double d) + { + debug << "Double(" << d << ")" << std::endl; + if (states.top() == State::IN_ERROR) { + debug << "In ERROR state" << std::endl; + return false; + } + if constexpr (!(std::is_same_v> || std::is_same_v>)) { + states.push(State::IN_ERROR); + return true; + } + if (states.top() == State::IN_ARRAY || states.top() == State::IN_ROW) { + if constexpr (std::is_same_v>) { + debug << "added to array as double" << std::endl; + accumulatedData.push_back(d); + return true; + } else if constexpr (std::is_same_v>) { + debug << "added to array as float" << std::endl; + accumulatedData.push_back(static_cast(d)); + return true; + } + } + states.push(State::IN_ERROR); + return true; + } + + bool Bool(bool b) + { + debug << "Bool(" << b << ")" << std::endl; + if (states.top() == State::IN_ERROR) { + debug << "In ERROR state" << std::endl; + return false; + } + if constexpr (!std::is_same_v>) { + states.push(State::IN_ERROR); + return false; + } else { + if (states.top() == State::IN_ARRAY) { + debug << "added to array" << std::endl; + accumulatedData.push_back(b); + return true; + } + states.push(State::IN_ERROR); + return true; + } + } + + bool String(const Ch* str, SizeType, bool) + { + debug << "String(" << str << ")" << std::endl; + if (states.top() == State::IN_ERROR) { + debug << "In ERROR state" << std::endl; + return false; + } + if constexpr (!(V == VariantType::ArrayString || isLabeledArray())) { + states.push(State::IN_ERROR); + return true; + } else { + if (states.top() == State::IN_ARRAY || states.top() == State::IN_ROW) { + debug << "added to array" << std::endl; + if constexpr (isLabeledArray()) { + if (currentKey == labels_rows_str) { + labels_rows.push_back(str); + return true; + } + if (currentKey == labels_cols_str) { + labels_cols.push_back(str); + return true; + } + } + if (currentKey == "values") { + if constexpr (std::is_same_v>) { + accumulatedData.push_back(str); + } else { + states.push(State::IN_ERROR); + } + return true; + } + return true; + } + states.push(State::IN_ERROR); + return true; + } + } + + bool StartObject() + { + debug << "StartObject()" << std::endl; + if (states.top() == State::IN_ERROR) { + debug << "In ERROR state" << std::endl; + return false; + } + if (states.top() == State::IN_START) { + states.push(State::IN_DATA); + return true; + } + states.push(State::IN_ERROR); + return true; + } + + bool Key(const Ch* str, SizeType, bool) + { + debug << "Key(" << str << ")" << std::endl; + if (states.top() == State::IN_ERROR) { + debug << "In ERROR state" << std::endl; + currentKey = str; + return false; + } + if (states.top() == State::IN_DATA) { + // no previous keys + states.push(State::IN_KEY); + currentKey = str; + return true; + } + if (states.top() == State::IN_KEY) { + currentKey = str; + if constexpr (!isLabeledArray()) { + debug << "extra keys in a single-key variant" << std::endl; + states.push(State::IN_ERROR); + return true; + } + return true; + } + currentKey = str; + states.push(State::IN_ERROR); + return true; + } + + bool EndObject(SizeType) + { + debug << "EndObject()" << std::endl; + if (states.top() == State::IN_ERROR) { + debug << "In ERROR state" << std::endl; + return false; + } + if (states.top() == State::IN_KEY) { + if constexpr (isArray()) { + debug << "creating 1d-array variant" << std::endl; + result = Variant(accumulatedData); + } else if constexpr (isArray2D()) { + debug << "creating 2d-array variant" << std::endl; + assert(accumulatedData.size() == rows * cols); + result = Variant(Array2D{accumulatedData, rows, cols}); + } else if constexpr (isLabeledArray()) { + debug << "creating labeled array variant" << std::endl; + assert(accumulatedData.size() == rows * cols); + if (labels_rows.empty() == false) { + assert(labels_rows.size() == rows); + } + if (labels_cols.empty() == false) { + assert(labels_cols.size() == cols); + } + result = Variant(LabeledArray{Array2D{accumulatedData, rows, cols}, labels_rows, labels_cols}); + } + states.push(State::IN_STOP); + return true; + } + states.push(State::IN_ERROR); + return true; + } + + bool StartArray() + { + debug << "StartArray()" << std::endl; + if (states.top() == State::IN_ERROR) { + debug << "In ERROR state" << std::endl; + return false; + } + if (states.top() == State::IN_KEY) { + states.push(State::IN_ARRAY); + return true; + } else if (states.top() == State::IN_ARRAY) { + if constexpr (isArray2D() || isLabeledArray()) { + states.push(State::IN_ROW); + return true; + } + } + states.push(State::IN_ERROR); + return true; + } + + bool EndArray(SizeType elementCount) + { + debug << "EndArray()" << std::endl; + if (states.top() == State::IN_ERROR) { + debug << "In ERROR state" << std::endl; + return false; + } + if (states.top() == State::IN_ARRAY) { + // finish up array + states.pop(); + if constexpr (isArray2D() || isLabeledArray()) { + rows = elementCount; + } + return true; + } else if (states.top() == State::IN_ROW) { + // finish up row + states.pop(); + if constexpr (isArray2D() || isLabeledArray()) { + cols = elementCount; + } + return true; + } + states.push(State::IN_ERROR); + return true; + } + + std::stack states; + std::ostringstream debug; + + uint32_t rows; + uint32_t cols; + std::string currentKey; + std::vector> accumulatedData; + std::vector labels_rows; + std::vector labels_cols; + Variant result; +}; +} // namespace + +template +Variant VariantJSONHelpers::read(std::istream& s) +{ + rapidjson::Reader reader; + rapidjson::IStreamWrapper isw(s); + VariantReader vreader; + bool ok = reader.Parse(isw, vreader); + + if (ok == false) { + std::stringstream error; + error << "Cannot parse serialized Variant, error: " << rapidjson::GetParseError_En(reader.GetParseErrorCode()) << " at offset: " << reader.GetErrorOffset(); + throw std::runtime_error(error.str()); + } + return vreader.result; +} + +template +void writeVariant(std::ostream& o, Variant const& v) +{ + if constexpr (isArray() || isArray2D() || isLabeledArray()) { + using type = variant_array_element_type_t; + rapidjson::OStreamWrapper osw(o); + rapidjson::Writer w(osw); + + auto writeArray = [&](auto* values, size_t size) { + using T = std::remove_pointer_t; + w.StartArray(); + for (auto i = 0u; i < size; ++i) { + if constexpr (std::is_same_v) { + w.Int(values[i]); + } else if constexpr (std::is_same_v || std::is_same_v) { + w.Double(values[i]); + } else if constexpr (std::is_same_v) { + w.Bool(values[i]); + } else if constexpr (std::is_same_v) { + w.String(values[i].c_str()); + } + } + w.EndArray(); + }; + + auto writeVector = [&](auto&& vector) { + return writeArray(vector.data(), vector.size()); + }; + + auto writeArray2D = [&](auto&& array2d) { + using T = typename std::decay_t::element_t; + w.StartArray(); + for (auto i = 0u; i < array2d.rows; ++i) { + w.StartArray(); + for (auto j = 0u; j < array2d.cols; ++j) { + if constexpr (std::is_same_v) { + w.Int(array2d(i, j)); + } else if constexpr (std::is_same_v || std::is_same_v) { + w.Double(array2d(i, j)); + } else if constexpr (std::is_same_v) { + w.String(array2d(i, j).c_str()); + } + } + w.EndArray(); + } + w.EndArray(); + }; + + auto writeLabeledArray = [&](auto&& array) { + w.Key(labels_rows_str); + writeVector(array.getLabelsRows()); + w.Key(labels_cols_str); + writeVector(array.getLabelsCols()); + w.Key("values"); + writeArray2D(array.getData()); + }; + + w.StartObject(); + if constexpr (isArray()) { + w.Key("values"); + writeArray(v.get(), v.size()); + } else if constexpr (isArray2D()) { + w.Key("values"); + writeArray2D(v.get>()); + } else if constexpr (isLabeledArray()) { + writeLabeledArray(v.get>()); + } else if constexpr (V == VariantType::Dict) { + // nothing to do for dicts + } + w.EndObject(); + } +} + +template Variant VariantJSONHelpers::read(std::istream& s); +template Variant VariantJSONHelpers::read(std::istream& s); +template Variant VariantJSONHelpers::read(std::istream& s); +template Variant VariantJSONHelpers::read(std::istream& s); +template Variant VariantJSONHelpers::read(std::istream& s); +template Variant VariantJSONHelpers::read(std::istream& s); +template Variant VariantJSONHelpers::read(std::istream& s); +template Variant VariantJSONHelpers::read(std::istream& s); +template Variant VariantJSONHelpers::read(std::istream& s); +template Variant VariantJSONHelpers::read(std::istream& s); +template Variant VariantJSONHelpers::read(std::istream& s); + +void VariantJSONHelpers::write(std::ostream& o, Variant const& v) +{ + switch (v.type()) { + case VariantType::ArrayInt: + writeVariant(o, v); + break; + case VariantType::ArrayFloat: + writeVariant(o, v); + break; + case VariantType::ArrayDouble: + writeVariant(o, v); + break; + case VariantType::ArrayBool: + throw std::runtime_error("Bool vectors not implemented yet"); + // writeVariant(o, v); + break; + case VariantType::ArrayString: + writeVariant(o, v); + break; + case VariantType::Array2DInt: + writeVariant(o, v); + break; + case VariantType::Array2DFloat: + writeVariant(o, v); + break; + case VariantType::Array2DDouble: + writeVariant(o, v); + break; + case VariantType::LabeledArrayInt: + writeVariant(o, v); + break; + case VariantType::LabeledArrayFloat: + writeVariant(o, v); + break; + case VariantType::LabeledArrayDouble: + writeVariant(o, v); + break; + case VariantType::LabeledArrayString: + writeVariant(o, v); + break; + case VariantType::Dict: + writeVariant(o, v); + default: + break; + } +} +} // namespace o2::framework