From 9e6c04831877311b1ec9c7fbc4f9f3edd11aec32 Mon Sep 17 00:00:00 2001 From: Anton Alkin Date: Thu, 3 Apr 2025 12:36:00 +0200 Subject: [PATCH 1/5] DPL Analysis: add configurable expression columns, tables and Defines<> template to create them --- Framework/Core/include/Framework/ASoA.h | 63 +++++++++++++++++- .../Core/include/Framework/AnalysisHelpers.h | 51 +++++++++++++++ .../Core/include/Framework/AnalysisManagers.h | 65 ++++++++++++------- .../Core/include/Framework/AnalysisTask.h | 6 +- .../Core/include/Framework/Expressions.h | 11 +++- .../Core/include/Framework/TableBuilder.h | 54 ++++++++++----- Framework/Core/src/TableBuilder.cxx | 9 ++- Framework/Core/src/runDataProcessing.cxx | 2 +- Framework/Core/test/test_TableSpawner.cxx | 24 +++++++ 9 files changed, 236 insertions(+), 49 deletions(-) diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h index f21decd0d5c45..b23a312948a11 100644 --- a/Framework/Core/include/Framework/ASoA.h +++ b/Framework/Core/include/Framework/ASoA.h @@ -425,6 +425,9 @@ concept has_metadata = is_metadata_trait && not_void; template concept has_extension = is_metadata && not_void; +template +concept has_configurable_extension = has_extension && requires(T t) { typename T::configurable_t; std::same_as; }; + template concept is_spawnable_column = std::same_as; @@ -2355,7 +2358,7 @@ O2HASH("TEST/0"); DECLARE_SOA_BITMAP_COLUMN_FULL(_Name_, _Getter_, _Size_, "f" #_Name_) /// An 'expression' column. i.e. a column that can be calculated from other -/// columns with gandiva based on supplied C++ expression. +/// columns with gandiva based on static C++ expression. #define DECLARE_SOA_EXPRESSION_COLUMN_FULL(_Name_, _Getter_, _Type_, _Label_, _Expression_) \ struct _Name_ : o2::soa::Column<_Type_, _Name_> { \ static constexpr const char* mLabel = _Label_; \ @@ -2393,6 +2396,38 @@ O2HASH("TEST/0"); #define DECLARE_SOA_EXPRESSION_COLUMN(_Name_, _Getter_, _Type_, _Expression_) \ DECLARE_SOA_EXPRESSION_COLUMN_FULL(_Name_, _Getter_, _Type_, "f" #_Name_, _Expression_); +/// A configurable 'expression' column. i.e. a column that can be calculated from other +/// columns with gandiva based on dynamically supplied C++ expression or a string definition. +#define DECLARE_SOA_CONFIGURABLE_EXPRESSION_COLUMN(_Name_, _Getter_, _Type_, _Label_) \ + struct _Name_ : o2::soa::Column<_Type_, _Name_> { \ + static constexpr const char* mLabel = _Label_; \ + static constexpr const int32_t mHash = _Label_ ""_h; \ + using base = o2::soa::Column<_Type_, _Name_>; \ + using type = _Type_; \ + using column_t = _Name_; \ + using spawnable_t = std::true_type; \ + _Name_(arrow::ChunkedArray const* column) \ + : o2::soa::Column<_Type_, _Name_>(o2::soa::ColumnIterator(column)) \ + { \ + } \ + \ + _Name_() = default; \ + _Name_(_Name_ const& other) = default; \ + _Name_& operator=(_Name_ const& other) = default; \ + \ + decltype(auto) _Getter_() const \ + { \ + return *mColumnIterator; \ + } \ + \ + decltype(auto) get() const \ + { \ + return _Getter_(); \ + } \ + }; \ + [[maybe_unused]] static constexpr o2::framework::expressions::BindingNode _Getter_ { _Label_, o2::framework::TypeIdHelpers::uniqueId<_Name_>(), \ + o2::framework::expressions::selectArrowType<_Type_>() } + /// An index column is a column of indices to elements / of another table named /// _Name_##s. The column name will be _Name_##Id and will always be stored in /// "fIndex"#_Table_#[_Suffix_]. If _Suffix_ is not empty it has to begin @@ -3104,6 +3139,32 @@ consteval auto getIndexTargets() O2HASH(#_Name_ "Extension"); \ DECLARE_SOA_EXTENDED_TABLE_FULL(_Name_, #_Name_ "Extension", _Table_, "AOD", "EX" _Description_, 0, __VA_ARGS__) +#define DECLARE_SOA_CONFIGURABLE_EXTENDED_TABLE_FULL(_Name_, _Label_, _OriginalTable_, _Origin_, _Desc_, _Version_, ...) \ + O2HASH(_Desc_ "/" #_Version_); \ + template \ + using _Name_##CfgExtensionFrom = soa::Table, o2::aod::Hash<_Desc_ "/" #_Version_ ""_h>, O>; \ + using _Name_##CfgExtension = _Name_##CfgExtensionFrom>; \ + template > \ + struct _Name_##CfgExtensionMetadataFrom : TableMetadata, __VA_ARGS__> { \ + using base_table_t = _OriginalTable_; \ + using extension_table_t = _Name_##CfgExtensionFrom; \ + using placeholders_pack_t = framework::pack<__VA_ARGS__>; \ + using configurable_t = std::true_type; \ + static constexpr auto sources = _OriginalTable_::originals; \ + }; \ + using _Name_##CfgExtensionMetadata = _Name_##CfgExtensionMetadataFrom>; \ + template <> \ + struct MetadataTrait> { \ + using metadata = _Name_##CfgExtensionMetadata; \ + }; \ + template \ + using _Name_##From = o2::soa::JoinFull, _OriginalTable_, _Name_##CfgExtensionFrom>; \ + using _Name_ = _Name_##From>; + +#define DECLARE_SOA_CONFIGURABLE_EXTENDED_TABLE(_Name_, _Table_, _Description_, ...) \ + O2HASH(#_Name_ "CfgExtension"); \ + DECLARE_SOA_CONFIGURABLE_EXTENDED_TABLE_FULL(_Name_, #_Name_ "CfgExtension", _Table_, "AOD", "EX" _Description_, 0, __VA_ARGS__) + #define DECLARE_SOA_INDEX_TABLE_FULL(_Name_, _Key_, _Origin_, _Version_, _Desc_, _Exclusive_, ...) \ O2HASH(#_Name_); \ O2HASH(_Desc_ "/" #_Version_); \ diff --git a/Framework/Core/include/Framework/AnalysisHelpers.h b/Framework/Core/include/Framework/AnalysisHelpers.h index 59bade6d43cd0..799bd0438c209 100644 --- a/Framework/Core/include/Framework/AnalysisHelpers.h +++ b/Framework/Core/include/Framework/AnalysisHelpers.h @@ -247,6 +247,9 @@ struct TableTransform { template concept is_spawnable = soa::has_metadata>> && soa::has_extension>::metadata>; +template +concept is_dynamically_spawnable = soa::has_metadata>> && soa::has_configurable_extension>::metadata>; + template constexpr auto transformBase() { @@ -282,12 +285,60 @@ struct Spawns : decltype(transformBase()) { } std::shared_ptr table = nullptr; std::shared_ptr extension = nullptr; + std::shared_ptr projector = nullptr; }; template concept is_spawns = requires(T t) { typename T::metadata; requires std::same_as; + requires std::same_as>; +}; + +/// This helper struct allows you to declare extended tables with dynamically-supplied +/// expressions to be created by the task +/// The actual expressions have to be set in init() for the configurable expression +/// columns, used to define the table + +template +struct Defines : decltype(transformBase()) { + using spawnable_t = T; + using metadata = decltype(transformBase())::metadata; + using extension_t = typename metadata::extension_table_t; + using base_table_t = typename metadata::base_table_t; + using placeholders_pack_t = typename metadata::placeholders_pack_t; + static constexpr size_t N = framework::pack_size(placeholders_pack_t{}); + + constexpr auto pack() + { + return placeholders_pack_t{}; + } + + typename T::table_t* operator->() + { + return table.get(); + } + typename T::table_t const& operator*() const + { + return *table; + } + + auto asArrowTable() + { + return extension->asArrowTable(); + } + std::shared_ptr table = nullptr; + std::shared_ptr extension = nullptr; + + std::array projectors; + std::shared_ptr projector = nullptr; +}; + +template +concept is_defines = requires(T t) { + typename T::metadata; + requires std::same_as; + requires std::same_as>; }; /// Policy to control index building diff --git a/Framework/Core/include/Framework/AnalysisManagers.h b/Framework/Core/include/Framework/AnalysisManagers.h index 733e91706b4a8..dbf8ebb5f439a 100644 --- a/Framework/Core/include/Framework/AnalysisManagers.h +++ b/Framework/Core/include/Framework/AnalysisManagers.h @@ -141,22 +141,27 @@ bool requestInputs(std::vector&, T const&) } template -bool requestInputs(std::vector& inputs, T const& spawns) -{ - auto base_specs = spawns.base_specs(); - for (auto base_spec : base_specs) { - base_spec.metadata.push_back(ConfigParamSpec{std::string{"control:spawn"}, VariantType::Bool, true, {"\"\""}}); - DataSpecUtils::updateInputList(inputs, std::forward(base_spec)); - } - return true; +const char* controlOption() { + return "control:spawn"; } template -bool requestInputs(std::vector& inputs, T const& builds) +const char* controlOption() { + return "control:build"; +} + +template +const char* controlOption() { + return "control:define"; +} + +template + requires(is_spawns || is_builds || is_defines) +bool requestInputs(std::vector& inputs, T const& entity) { - auto base_specs = builds.base_specs(); + auto base_specs = entity.base_specs(); for (auto base_spec : base_specs) { - base_spec.metadata.push_back(ConfigParamSpec{std::string{"control:build"}, VariantType::Bool, true, {"\"\""}}); + base_spec.metadata.push_back(ConfigParamSpec{std::string{controlOption()}, VariantType::Bool, true, {"\"\""}}); DataSpecUtils::updateInputList(inputs, std::forward(base_spec)); } return true; @@ -219,17 +224,11 @@ bool appendOutput(std::vector& outputs, T& obj, uint32_t hash) return true; } -template -bool appendOutput(std::vector& outputs, T& spawns, uint32_t) -{ - outputs.emplace_back(spawns.spec()); - return true; -} - -template -bool appendOutput(std::vector& outputs, T& builds, uint32_t) +template + requires(is_spawns || is_builds || is_defines) +bool appendOutput(std::vector& outputs, T& entity, uint32_t) { - outputs.emplace_back(builds.spec()); + outputs.emplace_back(entity.spec()); return true; } @@ -286,7 +285,7 @@ bool prepareOutput(ProcessingContext& context, T& spawns) originalTable = makeEmptyTable(o2::aod::label()); } - spawns.extension = std::make_shared(o2::framework::spawner>(originalTable, o2::aod::label())); + spawns.extension = std::make_shared(o2::framework::spawner>(originalTable, o2::aod::label(), spawns.projector)); spawns.table = std::make_shared(soa::ArrowHelpers::joinTables({spawns.extension->asArrowTable(), originalTable})); return true; } @@ -298,6 +297,21 @@ bool prepareOuput(ProcessingContext& context, T& builds) return builds.template build(builds.pack(), extractOriginals(context)); } +template +bool prepareOutput(ProcessingContext& context, T& defines) +{ + using metadata = o2::aod::MetadataTrait>::metadata; + auto originalTable = soa::ArrowHelpers::joinTables(extractOriginals(context)); + if (originalTable->schema()->fields().empty() == true) { + using base_table_t = typename T::base_table_t::table_t; + originalTable = makeEmptyTable(o2::aod::label()); + } + + defines.extension = std::make_shared(o2::framework::spawner>(originalTable, o2::aod::label(), defines.projectors.data(), defines.projector)); + defines.table = std::make_shared(soa::ArrowHelpers::joinTables({defines.extension->asArrowTable(), originalTable})); + return true; +} + template bool finalizeOutput(ProcessingContext&, T&) { @@ -333,6 +347,13 @@ bool finalizeOutput(ProcessingContext& context, T& builds) return true; } +template +bool finalizeOutput(ProcessingContext& context, T& defines) +{ + context.outputs().adopt(defines.output(), defines.asArrowTable()); + return true; +} + /// Service handling template bool addService(std::vector&, T&) diff --git a/Framework/Core/include/Framework/AnalysisTask.h b/Framework/Core/include/Framework/AnalysisTask.h index b78bf61aea558..bd1a1cfd88954 100644 --- a/Framework/Core/include/Framework/AnalysisTask.h +++ b/Framework/Core/include/Framework/AnalysisTask.h @@ -576,7 +576,7 @@ DataProcessorSpec adaptAnalysisTask(ConfigContext const& ctx, Args&&... args) homogeneous_apply_refs([&ic](auto&& element) { return analysis_task_parsers::prepareService(ic, element); }, *task.get()); auto& callbacks = ic.services().get(); - auto endofdatacb = [task](EndOfStreamContext& eosContext) { + auto eoscb = [task](EndOfStreamContext& eosContext) { homogeneous_apply_refs([&eosContext](auto& element) { analysis_task_parsers::postRunService(eosContext, element); analysis_task_parsers::postRunOutput(eosContext, element); @@ -585,13 +585,13 @@ DataProcessorSpec adaptAnalysisTask(ConfigContext const& ctx, Args&&... args) eosContext.services().get().readyToQuit(QuitRequest::Me); }; - callbacks.set(endofdatacb); + callbacks.set(eoscb); /// update configurables in filters and partitions homogeneous_apply_refs( [&ic](auto& element) -> bool { return analysis_task_parsers::updatePlaceholders(ic, element); }, *task.get()); - /// create for filters gandiva trees matched to schemas and store the pointers into expressionInfos + /// create expression trees for filters gandiva trees matched to schemas and store the pointers into expressionInfos homogeneous_apply_refs([&expressionInfos](auto& element) { return analysis_task_parsers::createExpressionTrees(expressionInfos, element); }, diff --git a/Framework/Core/include/Framework/Expressions.h b/Framework/Core/include/Framework/Expressions.h index ff22a35a00a23..14a6096560a9c 100644 --- a/Framework/Core/include/Framework/Expressions.h +++ b/Framework/Core/include/Framework/Expressions.h @@ -404,6 +404,8 @@ inline Node ifnode(Node&& condition_, Configurable const& then_, Configurabl /// A struct, containing the root of the expression tree struct Filter { + Filter() = default; + Filter(Node&& node_) : node{std::make_unique(std::forward(node_))} { (void)designateSubtrees(node.get()); @@ -413,7 +415,14 @@ struct Filter { { (void)designateSubtrees(node.get()); } - std::unique_ptr node; + + Filter& operator= (Filter&& other) noexcept + { + node = std::move(other.node); + return *this; + } + + std::unique_ptr node = nullptr; size_t designateSubtrees(Node* node, size_t index = 0); }; diff --git a/Framework/Core/include/Framework/TableBuilder.h b/Framework/Core/include/Framework/TableBuilder.h index df392f6fbbaf5..6ee2234372019 100644 --- a/Framework/Core/include/Framework/TableBuilder.h +++ b/Framework/Core/include/Framework/TableBuilder.h @@ -855,11 +855,41 @@ auto makeEmptyTable(const char* name, framework::pack p) } std::shared_ptr spawnerHelper(std::shared_ptr const& fullTable, std::shared_ptr newSchema, size_t nColumns, - expressions::Projector* projectors, std::vector> const& fields, const char* name); + expressions::Projector* projectors, std::vector> const& fields, const char* name, std::shared_ptr projector = nullptr); /// Expression-based column generator to materialize columns template -auto spawner(std::vector>&& tables, const char* name) + requires(soa::has_configurable_extension::metadata>) +auto spawner(std::vector>&& tables, const char* name, o2::framework::expressions::Projector* projectors, std::shared_ptr projector) +{ + using placeholders_pack_t = typename o2::aod::MetadataTrait::metadata::placeholders_pack_t; + auto fullTable = soa::ArrowHelpers::joinTables(std::move(tables)); + if (fullTable->num_rows() == 0) { + return makeEmptyTable(name, placeholders_pack_t{}); + } + static auto fields = o2::soa::createFieldsFromColumns(placeholders_pack_t{}); + static auto new_schema = std::make_shared(fields); + + return spawnerHelper(fullTable, new_schema, framework::pack_size(placeholders_pack_t{}), projectors, fields, name, projector); +} + +template + requires(soa::has_configurable_extension::metadata>) +auto spawner(std::shared_ptr const& fullTable, const char* name, o2::framework::expressions::Projector* projectors, std::shared_ptr projector) +{ + using placeholders_pack_t = typename o2::aod::MetadataTrait::metadata::placeholders_pack_t; + if (fullTable->num_rows() == 0) { + return makeEmptyTable(name, placeholders_pack_t{}); + } + static auto fields = o2::soa::createFieldsFromColumns(placeholders_pack_t{}); + static auto new_schema = std::make_shared(fields); + + return spawnerHelper(fullTable, new_schema, framework::pack_size(placeholders_pack_t{}), projectors, fields, name, projector); +} + +template + requires(soa::has_extension::metadata> && !soa::has_configurable_extension::metadata>) +auto spawner(std::vector>&& tables, const char* name, std::shared_ptr projector = nullptr) { using expression_pack_t = typename o2::aod::MetadataTrait::metadata::expression_pack_t; auto fullTable = soa::ArrowHelpers::joinTables(std::move(tables)); @@ -874,11 +904,12 @@ auto spawner(std::vector>&& tables, const char* na } (expression_pack_t{}); - return spawnerHelper(fullTable, new_schema, framework::pack_size(expression_pack_t{}), projectors.data(), fields, name); + return spawnerHelper(fullTable, new_schema, framework::pack_size(expression_pack_t{}), projectors.data(), fields, name, projector); } template -auto spawner(std::shared_ptr const& fullTable, const char* name) + requires(soa::has_extension::metadata> && !soa::has_configurable_extension::metadata>) +auto spawner(std::shared_ptr const& fullTable, const char* name, std::shared_ptr projector = nullptr) { using expression_pack_t = typename o2::aod::MetadataTrait::metadata::expression_pack_t; if (fullTable->num_rows() == 0) { @@ -892,22 +923,9 @@ auto spawner(std::shared_ptr const& fullTable, const char* name) } (expression_pack_t{}); - return spawnerHelper(fullTable, new_schema, framework::pack_size(expression_pack_t{}), projectors.data(), fields, name); + return spawnerHelper(fullTable, new_schema, framework::pack_size(expression_pack_t{}), projectors.data(), fields, name, projector); } -// template -// auto spawner(framework::pack columns, std::vector>&& tables, const char* name) -// { -// auto fullTable = soa::ArrowHelpers::joinTables(std::move(tables)); -// if (fullTable->num_rows() == 0) { -// return makeEmptyTable>(name); -// } -// static auto fields = o2::soa::createFieldsFromColumns(columns); -// static auto new_schema = std::make_shared(fields); -// std::array projectors{{std::move(C::Projector())...}}; -// return spawnerHelper(fullTable, new_schema, sizeof...(C), projectors.data(), fields, name); -// } - template auto spawner(framework::pack columns, std::vector>&& tables, const char* name) { diff --git a/Framework/Core/src/TableBuilder.cxx b/Framework/Core/src/TableBuilder.cxx index 5681568d40ebd..5ce9f6581587a 100644 --- a/Framework/Core/src/TableBuilder.cxx +++ b/Framework/Core/src/TableBuilder.cxx @@ -85,9 +85,12 @@ void TableBuilder::setLabel(const char* label) } std::shared_ptr spawnerHelper(std::shared_ptr const& fullTable, std::shared_ptr newSchema, size_t nColumns, - expressions::Projector* projectors, std::vector> const& fields, const char* name) + expressions::Projector* projectors, std::vector> const& fields, const char* name, + std::shared_ptr projector) { - auto mergedProjectors = framework::expressions::createProjectorHelper(nColumns, projectors, fullTable->schema(), fields); + if (projector == nullptr) { + projector = framework::expressions::createProjectorHelper(nColumns, projectors, fullTable->schema(), fields); + } arrow::TableBatchReader reader(*fullTable); std::shared_ptr batch; @@ -105,7 +108,7 @@ std::shared_ptr spawnerHelper(std::shared_ptr const& break; } try { - s = mergedProjectors->Evaluate(*batch, arrow::default_memory_pool(), &v); + s = projector->Evaluate(*batch, arrow::default_memory_pool(), &v); if (!s.ok()) { throw runtime_error_f("Cannot apply projector to source table of %s: %s", name, s.ToString().c_str()); } diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx index 6c38945039d84..a7e80134a2cc0 100644 --- a/Framework/Core/src/runDataProcessing.cxx +++ b/Framework/Core/src/runDataProcessing.cxx @@ -1684,7 +1684,7 @@ int runStateMachine(DataProcessorSpecs const& workflow, for (auto& input : device.inputs) { for (auto& param : input.metadata) { if (param.type == VariantType::Bool && param.name.find("control:") != std::string::npos) { - if (param.name != "control:default" && param.name != "control:spawn" && param.name != "control:build") { + if (param.name != "control:default" && param.name != "control:spawn" && param.name != "control:build" && param.name != "control:define") { auto confName = confNameFromParam(param.name).second; param.defaultValue = reg->get(confName.c_str()); } diff --git a/Framework/Core/test/test_TableSpawner.cxx b/Framework/Core/test/test_TableSpawner.cxx index 3d8879d115213..5bbb442e94764 100644 --- a/Framework/Core/test/test_TableSpawner.cxx +++ b/Framework/Core/test/test_TableSpawner.cxx @@ -28,10 +28,14 @@ DECLARE_SOA_COLUMN(Y, y, float); DECLARE_SOA_COLUMN(Z, z, float); DECLARE_SOA_EXPRESSION_COLUMN(Rsq, rsq, float, test::x* test::x + test::y * test::y + test::z * test::z); DECLARE_SOA_EXPRESSION_COLUMN(Sin, sin, float, test::x / nsqrt(test::x * test::x + test::y * test::y)); + +DECLARE_SOA_CONFIGURABLE_EXPRESSION_COLUMN(Cfg, cfg, float, "configurable"); } // namespace test DECLARE_SOA_TABLE(Points, "AOD", "PTSNG", test::X, test::Y, test::Z); DECLARE_SOA_EXTENDED_TABLE(ExPoints, Points, "EXPTSNG", 0, test::Rsq, test::Sin); + +DECLARE_SOA_CONFIGURABLE_EXTENDED_TABLE(ExcPoints, Points, "CFGPTS", test::Cfg); } // namespace o2::aod TEST_CASE("TestTableSpawner") @@ -71,4 +75,24 @@ TEST_CASE("TestTableSpawner") ++rexp; ++rexp_a; } + + Defines excpts; + excpts.projectors[0] = test::x* test::x + test::y * test::y + test::z * test::z; + + auto extension_2 = ExcPointsCfgExtension{o2::framework::spawner>({t1}, o2::aod::Hash<"ExcPoints"_h>::str, excpts.projectors.data(), excpts.projector)}; + auto excpoints = ExcPoints{{t1, extension_2.asArrowTable()}, 0}; + + rex = extension.begin(); + auto rex_2 = extension_2.begin(); + auto rexcp = excpoints.begin(); + + for (auto i = 1; i < 10; ++i) { + float rsq = i * i * 4 + i * i * 9 + i * i * 16; + REQUIRE(rex.rsq() == rsq); + REQUIRE(rex_2.cfg() == rsq); + REQUIRE(rexcp.cfg() == rsq); + ++rex; + ++rex_2; + ++rexcp; + } } From c142b97018a9f41ccaba2bdd6f333cabf6464fed Mon Sep 17 00:00:00 2001 From: ALICE Action Bot Date: Thu, 3 Apr 2025 12:20:55 +0000 Subject: [PATCH 2/5] Please consider the following formatting changes --- Framework/Core/include/Framework/ASoA.h | 2 +- Framework/Core/include/Framework/AnalysisManagers.h | 9 ++++++--- Framework/Core/include/Framework/Expressions.h | 2 +- Framework/Core/test/test_TableSpawner.cxx | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h index b23a312948a11..01f9eeb7bd66b 100644 --- a/Framework/Core/include/Framework/ASoA.h +++ b/Framework/Core/include/Framework/ASoA.h @@ -3162,7 +3162,7 @@ consteval auto getIndexTargets() using _Name_ = _Name_##From>; #define DECLARE_SOA_CONFIGURABLE_EXTENDED_TABLE(_Name_, _Table_, _Description_, ...) \ - O2HASH(#_Name_ "CfgExtension"); \ + O2HASH(#_Name_ "CfgExtension"); \ DECLARE_SOA_CONFIGURABLE_EXTENDED_TABLE_FULL(_Name_, #_Name_ "CfgExtension", _Table_, "AOD", "EX" _Description_, 0, __VA_ARGS__) #define DECLARE_SOA_INDEX_TABLE_FULL(_Name_, _Key_, _Origin_, _Version_, _Desc_, _Exclusive_, ...) \ diff --git a/Framework/Core/include/Framework/AnalysisManagers.h b/Framework/Core/include/Framework/AnalysisManagers.h index dbf8ebb5f439a..330eaf01f0be4 100644 --- a/Framework/Core/include/Framework/AnalysisManagers.h +++ b/Framework/Core/include/Framework/AnalysisManagers.h @@ -141,17 +141,20 @@ bool requestInputs(std::vector&, T const&) } template -const char* controlOption() { +const char* controlOption() +{ return "control:spawn"; } template -const char* controlOption() { +const char* controlOption() +{ return "control:build"; } template -const char* controlOption() { +const char* controlOption() +{ return "control:define"; } diff --git a/Framework/Core/include/Framework/Expressions.h b/Framework/Core/include/Framework/Expressions.h index 14a6096560a9c..1d2883418de71 100644 --- a/Framework/Core/include/Framework/Expressions.h +++ b/Framework/Core/include/Framework/Expressions.h @@ -416,7 +416,7 @@ struct Filter { (void)designateSubtrees(node.get()); } - Filter& operator= (Filter&& other) noexcept + Filter& operator=(Filter&& other) noexcept { node = std::move(other.node); return *this; diff --git a/Framework/Core/test/test_TableSpawner.cxx b/Framework/Core/test/test_TableSpawner.cxx index 5bbb442e94764..e67e62878bf59 100644 --- a/Framework/Core/test/test_TableSpawner.cxx +++ b/Framework/Core/test/test_TableSpawner.cxx @@ -77,7 +77,7 @@ TEST_CASE("TestTableSpawner") } Defines excpts; - excpts.projectors[0] = test::x* test::x + test::y * test::y + test::z * test::z; + excpts.projectors[0] = test::x * test::x + test::y * test::y + test::z * test::z; auto extension_2 = ExcPointsCfgExtension{o2::framework::spawner>({t1}, o2::aod::Hash<"ExcPoints"_h>::str, excpts.projectors.data(), excpts.projector)}; auto excpoints = ExcPoints{{t1, extension_2.asArrowTable()}, 0}; From add580c8738832eb779cec430bf445233ff79041 Mon Sep 17 00:00:00 2001 From: Anton Alkin Date: Thu, 3 Apr 2025 15:10:01 +0200 Subject: [PATCH 3/5] Update ASoA.h --- Framework/Core/include/Framework/ASoA.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h index 01f9eeb7bd66b..e098cd89f6d5d 100644 --- a/Framework/Core/include/Framework/ASoA.h +++ b/Framework/Core/include/Framework/ASoA.h @@ -426,7 +426,7 @@ template concept has_extension = is_metadata && not_void; template -concept has_configurable_extension = has_extension && requires(T t) { typename T::configurable_t; std::same_as; }; +concept has_configurable_extension = has_extension && requires(T t) { typename T::configurable_t; requires std::same_as; }; template concept is_spawnable_column = std::same_as; From d507b6ed5cf05551559a49ad3dc25c2000cc4baf Mon Sep 17 00:00:00 2001 From: Anton Alkin Date: Fri, 4 Apr 2025 09:39:24 +0200 Subject: [PATCH 4/5] improve caching behavior of projectors --- .../Core/include/Framework/AnalysisHelpers.h | 3 +- .../Core/include/Framework/TableBuilder.h | 14 +-- Framework/Core/src/AODReaderHelpers.cxx | 110 ++++++++++-------- Framework/Core/src/TableBuilder.cxx | 2 +- Framework/Core/test/test_TableSpawner.cxx | 4 +- 5 files changed, 76 insertions(+), 57 deletions(-) diff --git a/Framework/Core/include/Framework/AnalysisHelpers.h b/Framework/Core/include/Framework/AnalysisHelpers.h index 799bd0438c209..985f80cd548bc 100644 --- a/Framework/Core/include/Framework/AnalysisHelpers.h +++ b/Framework/Core/include/Framework/AnalysisHelpers.h @@ -795,7 +795,8 @@ template auto Extend(T const& table) { using output_t = Join, o2::aod::Hash<"JOIN/0"_h>, o2::aod::Hash<"JOIN"_h>, Cs...>>; - return output_t{{o2::framework::spawner(framework::pack{}, {table.asArrowTable()}, "dynamicExtension"), table.asArrowTable()}, 0}; + static std::shared_ptr projector = nullptr; + return output_t{{o2::framework::spawner(framework::pack{}, {table.asArrowTable()}, "dynamicExtension", projector), table.asArrowTable()}, 0}; } /// Template function to attach dynamic columns on-the-fly (e.g. inside diff --git a/Framework/Core/include/Framework/TableBuilder.h b/Framework/Core/include/Framework/TableBuilder.h index 6ee2234372019..32fe78b852eff 100644 --- a/Framework/Core/include/Framework/TableBuilder.h +++ b/Framework/Core/include/Framework/TableBuilder.h @@ -855,12 +855,12 @@ auto makeEmptyTable(const char* name, framework::pack p) } std::shared_ptr spawnerHelper(std::shared_ptr const& fullTable, std::shared_ptr newSchema, size_t nColumns, - expressions::Projector* projectors, std::vector> const& fields, const char* name, std::shared_ptr projector = nullptr); + expressions::Projector* projectors, std::vector> const& fields, const char* name, std::shared_ptr& projector); /// Expression-based column generator to materialize columns template requires(soa::has_configurable_extension::metadata>) -auto spawner(std::vector>&& tables, const char* name, o2::framework::expressions::Projector* projectors, std::shared_ptr projector) +auto spawner(std::vector>&& tables, const char* name, o2::framework::expressions::Projector* projectors, std::shared_ptr& projector) { using placeholders_pack_t = typename o2::aod::MetadataTrait::metadata::placeholders_pack_t; auto fullTable = soa::ArrowHelpers::joinTables(std::move(tables)); @@ -875,7 +875,7 @@ auto spawner(std::vector>&& tables, const char* na template requires(soa::has_configurable_extension::metadata>) -auto spawner(std::shared_ptr const& fullTable, const char* name, o2::framework::expressions::Projector* projectors, std::shared_ptr projector) +auto spawner(std::shared_ptr const& fullTable, const char* name, o2::framework::expressions::Projector* projectors, std::shared_ptr& projector) { using placeholders_pack_t = typename o2::aod::MetadataTrait::metadata::placeholders_pack_t; if (fullTable->num_rows() == 0) { @@ -889,7 +889,7 @@ auto spawner(std::shared_ptr const& fullTable, const char* name, o template requires(soa::has_extension::metadata> && !soa::has_configurable_extension::metadata>) -auto spawner(std::vector>&& tables, const char* name, std::shared_ptr projector = nullptr) +auto spawner(std::vector>&& tables, const char* name, std::shared_ptr& projector) { using expression_pack_t = typename o2::aod::MetadataTrait::metadata::expression_pack_t; auto fullTable = soa::ArrowHelpers::joinTables(std::move(tables)); @@ -909,7 +909,7 @@ auto spawner(std::vector>&& tables, const char* na template requires(soa::has_extension::metadata> && !soa::has_configurable_extension::metadata>) -auto spawner(std::shared_ptr const& fullTable, const char* name, std::shared_ptr projector = nullptr) +auto spawner(std::shared_ptr const& fullTable, const char* name, std::shared_ptr& projector) { using expression_pack_t = typename o2::aod::MetadataTrait::metadata::expression_pack_t; if (fullTable->num_rows() == 0) { @@ -927,7 +927,7 @@ auto spawner(std::shared_ptr const& fullTable, const char* name, s } template -auto spawner(framework::pack columns, std::vector>&& tables, const char* name) +auto spawner(framework::pack columns, std::vector>&& tables, const char* name, std::shared_ptr& projector) { auto fullTable = soa::ArrowHelpers::joinTables(std::move(tables)); if (fullTable->num_rows() == 0) { @@ -936,7 +936,7 @@ auto spawner(framework::pack columns, std::vector(fields); std::array projectors{{std::move(C::Projector())...}}; - return spawnerHelper(fullTable, new_schema, sizeof...(C), projectors.data(), fields, name); + return spawnerHelper(fullTable, new_schema, sizeof...(C), projectors.data(), fields, name, projector); } template diff --git a/Framework/Core/src/AODReaderHelpers.cxx b/Framework/Core/src/AODReaderHelpers.cxx index 6270d07a022e8..d570ebccce263 100644 --- a/Framework/Core/src/AODReaderHelpers.cxx +++ b/Framework/Core/src/AODReaderHelpers.cxx @@ -83,53 +83,64 @@ static inline auto extractOriginals(ProcessingContext& pc) return {pc.inputs().get(o2::aod::label())->asArrowTable()...}; }(std::make_index_sequence()); } +namespace { +template + requires(D::exclusive) +auto make_build(D metadata, InputSpec const& input, ProcessingContext& pc) +{ + using metadata_t = decltype(metadata); + using Key = typename metadata_t::Key; + using index_pack_t = typename metadata_t::index_pack_t; + constexpr auto sources = metadata_t::sources; + return o2::framework::IndexBuilder::indexBuilder(input.binding.c_str(), + extractOriginals(pc), + index_pack_t{}); +} + +template + requires(!D::exclusive) +auto make_build(D metadata, InputSpec const& input, ProcessingContext& pc) +{ + using metadata_t = decltype(metadata); + using Key = typename metadata_t::Key; + using index_pack_t = typename metadata_t::index_pack_t; + constexpr auto sources = metadata_t::sources; + return o2::framework::IndexBuilder::indexBuilder(input.binding.c_str(), + extractOriginals(pc), + index_pack_t{}); +} +} AlgorithmSpec AODReaderHelpers::indexBuilderCallback(std::vector& requested) { - return AlgorithmSpec::InitCallback{[requested](InitContext& ic) { + return AlgorithmSpec::InitCallback{[requested](InitContext& /*ic*/) { return [requested](ProcessingContext& pc) { auto outputs = pc.outputs(); // spawn tables for (auto& input : requested) { auto&& [origin, description, version] = DataSpecUtils::asConcreteDataMatcher(input); - auto maker = [&](auto metadata) { - using metadata_t = decltype(metadata); - using Key = typename metadata_t::Key; - using index_pack_t = typename metadata_t::index_pack_t; - constexpr auto sources = metadata_t::sources; - if constexpr (metadata_t::exclusive == true) { - return o2::framework::IndexBuilder::indexBuilder(input.binding.c_str(), - extractOriginals(pc), - index_pack_t{}); - } else { - return o2::framework::IndexBuilder::indexBuilder(input.binding.c_str(), - extractOriginals(pc), - index_pack_t{}); - } - }; - if (description == header::DataDescription{"MA_RN2_EX"}) { - outputs.adopt(Output{origin, description, version}, maker(o2::aod::Run2MatchedExclusiveMetadata{})); + outputs.adopt(Output{origin, description, version}, make_build(o2::aod::Run2MatchedExclusiveMetadata{}, input, pc)); } else if (description == header::DataDescription{"MA_RN2_SP"}) { - outputs.adopt(Output{origin, description, version}, maker(o2::aod::Run2MatchedSparseMetadata{})); + outputs.adopt(Output{origin, description, version}, make_build(o2::aod::Run2MatchedSparseMetadata{}, input, pc)); } else if (description == header::DataDescription{"MA_RN3_EX"}) { - outputs.adopt(Output{origin, description, version}, maker(o2::aod::Run3MatchedExclusiveMetadata{})); + outputs.adopt(Output{origin, description, version}, make_build(o2::aod::Run3MatchedExclusiveMetadata{}, input, pc)); } else if (description == header::DataDescription{"MA_RN3_SP"}) { - outputs.adopt(Output{origin, description, version}, maker(o2::aod::Run3MatchedSparseMetadata{})); + outputs.adopt(Output{origin, description, version}, make_build(o2::aod::Run3MatchedSparseMetadata{}, input, pc)); } else if (description == header::DataDescription{"MA_BCCOL_EX"}) { - outputs.adopt(Output{origin, description, version}, maker(o2::aod::MatchedBCCollisionsExclusiveMetadata{})); + outputs.adopt(Output{origin, description, version}, make_build(o2::aod::MatchedBCCollisionsExclusiveMetadata{}, input, pc)); } else if (description == header::DataDescription{"MA_BCCOL_SP"}) { - outputs.adopt(Output{origin, description, version}, maker(o2::aod::MatchedBCCollisionsSparseMetadata{})); + outputs.adopt(Output{origin, description, version}, make_build(o2::aod::MatchedBCCollisionsSparseMetadata{}, input, pc)); } else if (description == header::DataDescription{"MA_BCCOLS_EX"}) { - outputs.adopt(Output{origin, description, version}, maker(o2::aod::MatchedBCCollisionsExclusiveMultiMetadata{})); + outputs.adopt(Output{origin, description, version}, make_build(o2::aod::MatchedBCCollisionsExclusiveMultiMetadata{}, input, pc)); } else if (description == header::DataDescription{"MA_BCCOLS_SP"}) { - outputs.adopt(Output{origin, description, version}, maker(o2::aod::MatchedBCCollisionsSparseMultiMetadata{})); + outputs.adopt(Output{origin, description, version}, make_build(o2::aod::MatchedBCCollisionsSparseMultiMetadata{}, input, pc)); } else if (description == header::DataDescription{"MA_RN3_BC_SP"}) { - outputs.adopt(Output{origin, description, version}, maker(o2::aod::Run3MatchedToBCSparseMetadata{})); + outputs.adopt(Output{origin, description, version}, make_build(o2::aod::Run3MatchedToBCSparseMetadata{}, input, pc)); } else if (description == header::DataDescription{"MA_RN3_BC_EX"}) { - outputs.adopt(Output{origin, description, version}, maker(o2::aod::Run3MatchedToBCExclusiveMetadata{})); + outputs.adopt(Output{origin, description, version}, make_build(o2::aod::Run3MatchedToBCExclusiveMetadata{}, input, pc)); } else if (description == header::DataDescription{"MA_RN2_BC_SP"}) { - outputs.adopt(Output{origin, description, version}, maker(o2::aod::Run2MatchedToBCSparseMetadata{})); + outputs.adopt(Output{origin, description, version}, make_build(o2::aod::Run2MatchedToBCSparseMetadata{}, input, pc)); } else { throw std::runtime_error("Not an index table"); } @@ -138,6 +149,17 @@ AlgorithmSpec AODReaderHelpers::indexBuilderCallback(std::vector& req }}; } +namespace{ +template +auto make_spawn(InputSpec const& input, ProcessingContext& pc) +{ + using metadata_t = o2::aod::MetadataTrait::metadata; + constexpr auto sources = metadata_t::sources; + static std::shared_ptr projector = nullptr; + return o2::framework::spawner(extractOriginals(pc), input.binding.c_str(), projector); +} +} + AlgorithmSpec AODReaderHelpers::aodSpawnerCallback(std::vector& requested) { return AlgorithmSpec::InitCallback{[requested](InitContext& /*ic*/) { @@ -146,43 +168,37 @@ AlgorithmSpec AODReaderHelpers::aodSpawnerCallback(std::vector& reque // spawn tables for (auto& input : requested) { auto&& [origin, description, version] = DataSpecUtils::asConcreteDataMatcher(input); - auto maker = [&]() { - using metadata_t = o2::aod::MetadataTrait::metadata; - constexpr auto sources = metadata_t::sources; - return o2::framework::spawner(extractOriginals(pc), input.binding.c_str()); - }; - if (description == header::DataDescription{"EXTRACK"}) { - outputs.adopt(Output{origin, description, version}, maker.template operator()>()); + outputs.adopt(Output{origin, description, version}, make_spawn>(input, pc)); } else if (description == header::DataDescription{"EXTRACK_IU"}) { - outputs.adopt(Output{origin, description, version}, maker.template operator()>()); + outputs.adopt(Output{origin, description, version}, make_spawn>(input, pc)); } else if (description == header::DataDescription{"EXTRACKCOV"}) { - outputs.adopt(Output{origin, description, version}, maker.template operator()>()); + outputs.adopt(Output{origin, description, version}, make_spawn>(input, pc)); } else if (description == header::DataDescription{"EXTRACKCOV_IU"}) { - outputs.adopt(Output{origin, description, version}, maker.template operator()>()); + outputs.adopt(Output{origin, description, version}, make_spawn>(input, pc)); } else if (description == header::DataDescription{"EXTRACKEXTRA"}) { if (version == 0U) { - outputs.adopt(Output{origin, description, version}, maker.template operator()>()); + outputs.adopt(Output{origin, description, version}, make_spawn>(input, pc)); } else if (version == 1U) { - outputs.adopt(Output{origin, description, version}, maker.template operator()>()); + outputs.adopt(Output{origin, description, version}, make_spawn>(input, pc)); } else if (version == 2U) { - outputs.adopt(Output{origin, description, version}, maker.template operator()>()); + outputs.adopt(Output{origin, description, version}, make_spawn>(input, pc)); } } else if (description == header::DataDescription{"EXMFTTRACK"}) { if (version == 0U) { - outputs.adopt(Output{origin, description, version}, maker.template operator()>()); + outputs.adopt(Output{origin, description, version}, make_spawn>(input, pc)); } else if (version == 1U) { - outputs.adopt(Output{origin, description, version}, maker.template operator()>()); + outputs.adopt(Output{origin, description, version}, make_spawn>(input, pc)); } } else if (description == header::DataDescription{"EXFWDTRACK"}) { - outputs.adopt(Output{origin, description, version}, maker.template operator()>()); + outputs.adopt(Output{origin, description, version}, make_spawn>(input, pc)); } else if (description == header::DataDescription{"EXFWDTRACKCOV"}) { - outputs.adopt(Output{origin, description, version}, maker.template operator()>()); + outputs.adopt(Output{origin, description, version}, make_spawn>(input, pc)); } else if (description == header::DataDescription{"EXMCPARTICLE"}) { if (version == 0U) { - outputs.adopt(Output{origin, description, version}, maker.template operator()>()); + outputs.adopt(Output{origin, description, version}, make_spawn>(input, pc)); } else if (version == 1U) { - outputs.adopt(Output{origin, description, version}, maker.template operator()>()); + outputs.adopt(Output{origin, description, version}, make_spawn>(input, pc)); } } else { throw runtime_error("Not an extended table"); diff --git a/Framework/Core/src/TableBuilder.cxx b/Framework/Core/src/TableBuilder.cxx index 5ce9f6581587a..d9827559c2148 100644 --- a/Framework/Core/src/TableBuilder.cxx +++ b/Framework/Core/src/TableBuilder.cxx @@ -86,7 +86,7 @@ void TableBuilder::setLabel(const char* label) std::shared_ptr spawnerHelper(std::shared_ptr const& fullTable, std::shared_ptr newSchema, size_t nColumns, expressions::Projector* projectors, std::vector> const& fields, const char* name, - std::shared_ptr projector) + std::shared_ptr& projector) { if (projector == nullptr) { projector = framework::expressions::createProjectorHelper(nColumns, projectors, fullTable->schema(), fields); diff --git a/Framework/Core/test/test_TableSpawner.cxx b/Framework/Core/test/test_TableSpawner.cxx index e67e62878bf59..2291ba5f4f787 100644 --- a/Framework/Core/test/test_TableSpawner.cxx +++ b/Framework/Core/test/test_TableSpawner.cxx @@ -50,8 +50,10 @@ TEST_CASE("TestTableSpawner") auto t1 = b1.finalize(); Points st1{t1}; + std::shared_ptr projector = nullptr; + auto expoints_a = o2::soa::Extend(st1); - auto extension = ExPointsExtension{o2::framework::spawner>(t1, o2::aod::Hash<"ExPoints"_h>::str)}; + auto extension = ExPointsExtension{o2::framework::spawner>(t1, o2::aod::Hash<"ExPoints"_h>::str, projector)}; auto expoints = ExPoints{{t1, extension.asArrowTable()}, 0}; REQUIRE(expoints_a.size() == 9); From 153b261ae1fa44c8d63f07ff8c8f25b8766ff9fb Mon Sep 17 00:00:00 2001 From: ALICE Action Bot Date: Fri, 4 Apr 2025 07:48:01 +0000 Subject: [PATCH 5/5] Please consider the following formatting changes --- Framework/Core/src/AODReaderHelpers.cxx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Framework/Core/src/AODReaderHelpers.cxx b/Framework/Core/src/AODReaderHelpers.cxx index d570ebccce263..c413f2520919d 100644 --- a/Framework/Core/src/AODReaderHelpers.cxx +++ b/Framework/Core/src/AODReaderHelpers.cxx @@ -83,7 +83,8 @@ static inline auto extractOriginals(ProcessingContext& pc) return {pc.inputs().get(o2::aod::label())->asArrowTable()...}; }(std::make_index_sequence()); } -namespace { +namespace +{ template requires(D::exclusive) auto make_build(D metadata, InputSpec const& input, ProcessingContext& pc) @@ -109,7 +110,7 @@ auto make_build(D metadata, InputSpec const& input, ProcessingContext& pc) extractOriginals(pc), index_pack_t{}); } -} +} // namespace AlgorithmSpec AODReaderHelpers::indexBuilderCallback(std::vector& requested) { @@ -149,7 +150,8 @@ AlgorithmSpec AODReaderHelpers::indexBuilderCallback(std::vector& req }}; } -namespace{ +namespace +{ template auto make_spawn(InputSpec const& input, ProcessingContext& pc) { @@ -158,7 +160,7 @@ auto make_spawn(InputSpec const& input, ProcessingContext& pc) static std::shared_ptr projector = nullptr; return o2::framework::spawner(extractOriginals(pc), input.binding.c_str(), projector); } -} +} // namespace AlgorithmSpec AODReaderHelpers::aodSpawnerCallback(std::vector& requested) {