diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx index f8a9705e4eb62..85ed9cd573d8a 100644 --- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx +++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx @@ -14,7 +14,6 @@ #include "Framework/TableTreeHelpers.h" #include "Framework/AnalysisHelpers.h" #include "Framework/DataProcessingStats.h" -#include "Framework/RootTableBuilderHelpers.h" #include "Framework/RootArrowFilesystem.h" #include "Framework/AlgorithmSpec.h" #include "Framework/ConfigParamRegistry.h" diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt index f059984b5d85d..2691d9d33a0c6 100644 --- a/Framework/Core/CMakeLists.txt +++ b/Framework/Core/CMakeLists.txt @@ -328,11 +328,6 @@ o2_add_executable(dpl-run PUBLIC_LINK_LIBRARIES O2::Framework ) -o2_add_executable(verify-aod-file - SOURCES src/verifyAODFile.cxx - PUBLIC_LINK_LIBRARIES O2::Framework ROOT::TreePlayer - COMPONENT_NAME Framework) - # benchmarks foreach(b diff --git a/Framework/Core/include/Framework/RootTableBuilderHelpers.h b/Framework/Core/include/Framework/RootTableBuilderHelpers.h deleted file mode 100644 index 0fa818084a5a4..0000000000000 --- a/Framework/Core/include/Framework/RootTableBuilderHelpers.h +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#ifndef o2_framework_RootTableBuilderHelpers_H_INCLUDED -#define o2_framework_RootTableBuilderHelpers_H_INCLUDED - -#include "Framework/TableBuilder.h" -#include "Framework/Logger.h" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace o2::framework -{ - -template -struct TreeReaderValueTraits { -}; - -/// Trait class to go from a set of TTreeReaderValues to -/// arrow types. -template -struct TreeReaderValueTraits> { - using Type = typename TTreeReaderValue::NonConstT_t; - using ArrowType = typename o2::framework::detail::ConversionTraits::ArrowType; - using BuilderType = typename arrow::TypeTraits::BuilderType; -}; - -template -struct TreeReaderValueTraits> { - using Iterator = typename TTreeReaderArray::iterator; - using Type = std::pair; - using ArrowType = arrow::ListType; -}; - -static constexpr int PREBUFFER_SIZE = 32 * 1024; - -// When reading from a ROOT file special care must happen -// because uint64_t is platform specific while ULong64_t is -// always long long unsigned int (same for the signed version). -// By using this traits we make sure that any 64 bit quantity -// read from a root file uses the ROOT datatype, not the platform one. -template -struct Remap64Bit { - using type = T; -}; - -template <> -struct Remap64Bit { - using type = Long64_t; -}; - -template <> -struct Remap64Bit { - using type = ULong64_t; -}; - -template -struct Remap64Bit { - using type = Long64_t[N]; -}; - -template -struct Remap64Bit { - using type = ULong64_t[N]; -}; - -template -using Remap64Bit_t = typename Remap64Bit::type; - -template -struct ReaderHolder { - using Reader = TTreeReaderValue; - using Type = T; - - ReaderHolder(TBranch* branch, std::unique_ptr reader_) - : reader{std::move(reader_)} - { - } - - ReaderHolder(ReaderHolder&& other) - : reader{std::move(other.reader)}, - pos{other.pos} - { - } - - ReaderHolder& operator=(ReaderHolder&& other) = delete; - - std::unique_ptr reader; - int pos = 0; - Remap64Bit_t buffer[PREBUFFER_SIZE]; - int itemSize = sizeof(T); -}; - -template -struct ReaderHolder { - using Reader = TTreeReaderArray; - using Type = T (&)[N]; - - ReaderHolder(TBranch* branch, std::unique_ptr reader_) - : reader{std::move(reader_)} - { - } - - ReaderHolder(ReaderHolder&& other) - : reader{std::move(other.reader)}, - pos{other.pos} - { - } - - ReaderHolder& operator=(ReaderHolder&& other) = delete; - - std::unique_ptr reader; - int pos = 0; - Remap64Bit_t buffer[PREBUFFER_SIZE * N]; - int itemSize = sizeof(T) * N; -}; - -struct BulkExtractor { - template - static auto deref(ReaderHolder& holder, size_t maxSize) - { - holder.buffer[holder.pos % PREBUFFER_SIZE] = **holder.reader; - holder.pos++; - if (holder.pos == maxSize) { - return BulkInfo const*>{holder.buffer, maxSize % PREBUFFER_SIZE}; - } - // We flush only after PREBUFFER_SIZE items have been inserted - if ((holder.pos % PREBUFFER_SIZE) != 0) { - return BulkInfo const*>{nullptr, 0}; - } - return BulkInfo const*>{holder.buffer, PREBUFFER_SIZE}; - } - - template - static auto deref(ReaderHolder& holder, size_t maxSize) - { - memcpy(&holder.buffer[(holder.pos % PREBUFFER_SIZE) * N], &((*holder.reader)[0]), N * sizeof(T)); - holder.pos++; - if (holder.pos == maxSize) { - return BulkInfo const*>{holder.buffer, maxSize % PREBUFFER_SIZE}; - } - // We flush only after PREBUFFER_SIZE items have been inserted - if ((holder.pos % PREBUFFER_SIZE) != 0) { - return BulkInfo const*>{nullptr, 0}; - } - return BulkInfo const*>{reinterpret_cast(holder.buffer), PREBUFFER_SIZE}; - } -}; - -template -struct HolderMaker { - static auto make(TTreeReader& reader, char const* branchName) - { - using Reader = TTreeReaderValue; - return ReaderHolder{reader.GetTree()->GetBranch(branchName), std::move(std::make_unique(reader, branchName))}; - } -}; - -template -struct HolderMaker { - static auto make(TTreeReader& reader, char const* branchName) - { - using Reader = TTreeReaderArray; - return ReaderHolder{reader.GetTree()->GetBranch(branchName), std::move(std::make_unique(reader, branchName))}; - } -}; - -template -struct ColumnReaderTrait { - static auto createReader(TTreeReader& reader) - { - return HolderMaker>::make(reader, C::base::columnLabel()); - } -}; - -struct RootTableBuilderHelpers { - /// Use bulk insertion when TTreeReaderValue everywhere - template - static void convertTTree(TableBuilder& builder, - TTreeReader& reader, - ReaderHolder... holders) - { - std::array branchNames = {holders.reader->GetBranchName()...}; - TTree* tree = reader.GetTree(); - size_t maxExtries = reader.GetEntries(true); - tree->SetCacheSize(maxExtries * (holders.itemSize + ...)); - (tree->AddBranchToCache(tree->GetBranch(holders.reader->GetBranchName()), true), ...); - tree->StopCacheLearningPhase(); - - auto filler = builder.bulkPersistChunked::Type>...>(branchNames, maxExtries); - while (reader.Next()) { - filler(0, BulkExtractor::deref(holders, maxExtries)...); - } - } - - template - static void convertASoAColumns(TableBuilder& builder, TTreeReader& reader, pack) - { - return convertTTree(builder, reader, ColumnReaderTrait::createReader(reader)...); - } - - template - static void convertASoA(TableBuilder& builder, TTreeReader& reader) - { - return convertASoAColumns(builder, reader, typename T::persistent_columns_t{}); - } -}; - -} // namespace o2 -#endif // FRAMEWORK_ROOTTABLEBUILDERHELPERS_H diff --git a/Framework/Core/src/AODReaderHelpers.cxx b/Framework/Core/src/AODReaderHelpers.cxx index f6513b5facea3..6270d07a022e8 100644 --- a/Framework/Core/src/AODReaderHelpers.cxx +++ b/Framework/Core/src/AODReaderHelpers.cxx @@ -15,7 +15,6 @@ #include "Framework/AnalysisDataModelHelpers.h" #include "Framework/DataProcessingHelpers.h" #include "Framework/ExpressionHelpers.h" -#include "Framework/RootTableBuilderHelpers.h" #include "Framework/AlgorithmSpec.h" #include "Framework/ConfigParamRegistry.h" #include "Framework/ControlService.h" diff --git a/Framework/Core/src/verifyAODFile.cxx b/Framework/Core/src/verifyAODFile.cxx deleted file mode 100644 index 2660019031946..0000000000000 --- a/Framework/Core/src/verifyAODFile.cxx +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#include "Framework/AnalysisDataModel.h" -#include "Framework/RootTableBuilderHelpers.h" -#include "Framework/Logger.h" -#include "Framework/ASoA.h" -#include -#include -#include - -using namespace o2::framework; -using namespace o2::soa; - -template -void verifyTable(TFile* infile, const char* branchName) -{ - std::cout << "Table: " << o2::aod::label() << std::endl; - std::unique_ptr reader = std::make_unique(branchName, infile); - TableBuilder builder; - RootTableBuilderHelpers::convertASoA(builder, *reader); - auto table = builder.finalize(); - std::cout << table->schema()->ToString() << std::endl; - std::cout << "---" << std::endl; -} - -int main(int argc, char** argv) -{ - if (argc != 2) { - LOG(error) << "Bad number of arguments"; - return 1; - } - auto infile = std::make_unique(argv[1]); - if (infile.get() == nullptr || infile->IsOpen() == false) { - LOG(error) << "File not found: " << argv[1]; - return 1; - } - - verifyTable(infile.get(), "O2collision"); - verifyTable(infile.get(), "O2track"); - verifyTable(infile.get(), "O2track"); - verifyTable(infile.get(), "O2track"); - verifyTable(infile.get(), "O2calo"); - verifyTable(infile.get(), "O2fwdtrack"); - return 0; -} diff --git a/Framework/Core/test/test_Root2ArrowTable.cxx b/Framework/Core/test/test_Root2ArrowTable.cxx index 663be91a1e6f3..395048ae916d6 100644 --- a/Framework/Core/test/test_Root2ArrowTable.cxx +++ b/Framework/Core/test/test_Root2ArrowTable.cxx @@ -12,7 +12,6 @@ #include #include "Framework/TableBuilder.h" -#include "Framework/RootTableBuilderHelpers.h" #include "Framework/ASoA.h" #include "Framework/PluginManager.h" #include "../src/ArrowDebugHelpers.h" @@ -50,87 +49,6 @@ using namespace o2::framework; -TEST_CASE("RootTree2Table") -{ - using namespace o2::framework; - /// Create a simple TTree - TTree t1("t1", "a simple Tree with simple variables"); - Float_t xyz[3]; - Int_t ij[2]; - Float_t px, py, pz; - Double_t random; - Int_t ev; - t1.Branch("px", &px, "px/F"); - t1.Branch("py", &py, "py/F"); - t1.Branch("pz", &pz, "pz/F"); - t1.Branch("random", &random, "random/D"); - t1.Branch("ev", &ev, "ev/I"); - t1.Branch("xyz", xyz, "xyz[3]/F"); - t1.Branch("ij", ij, "ij[2]/I"); - // fill the tree - for (Int_t i = 0; i < 1000; i++) { - xyz[0] = 1; - xyz[1] = 2; - xyz[2] = 3; - gRandom->Rannor(px, py); - pz = px * px + py * py; - xyz[2] = i + 1; - ij[0] = i; - ij[1] = i + 1; - random = gRandom->Rndm(); - ev = i + 1; - t1.Fill(); - } - - // Create an arrow table from this. - TableBuilder builder; - TTreeReader reader(&t1); - auto&& xyzReader = HolderMaker::make(reader, "xyz"); - auto&& ijkReader = HolderMaker::make(reader, "ij"); - auto&& pxReader = HolderMaker::make(reader, "px"); - auto&& pyReader = HolderMaker::make(reader, "py"); - auto&& pzReader = HolderMaker::make(reader, "pz"); - auto&& randomReader = HolderMaker::make(reader, "random"); - auto&& evReader = HolderMaker::make(reader, "ev"); - - RootTableBuilderHelpers::convertTTree(builder, reader, std::move(xyzReader), std::move(ijkReader), std::move(pxReader), std::move(pyReader), std::move(pzReader), std::move(randomReader), std::move(evReader)); - auto table = builder.finalize(); - REQUIRE(table->num_rows() == 1000); - REQUIRE(table->num_columns() == 7); - REQUIRE(table->schema()->field(0)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id()); - REQUIRE(table->schema()->field(1)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id()); - REQUIRE(table->schema()->field(2)->type()->id() == arrow::float32()->id()); - REQUIRE(table->schema()->field(3)->type()->id() == arrow::float32()->id()); - REQUIRE(table->schema()->field(4)->type()->id() == arrow::float32()->id()); - REQUIRE(table->schema()->field(5)->type()->id() == arrow::float64()->id()); - REQUIRE(table->schema()->field(6)->type()->id() == arrow::int32()->id()); - - { - auto chunkToUse = table->column(0)->chunk(0); - chunkToUse = std::dynamic_pointer_cast(chunkToUse)->values(); - auto array = std::static_pointer_cast(chunkToUse); - // array of 3 floats, time 1000. - REQUIRE(array->length() == 3000); - const float* c = reinterpret_cast(array->values()->data()); - - CHECK(c[0] == 1); - CHECK(c[1] == 2); - CHECK(c[2] == 1); - } - { - auto chunkToUse = table->column(1)->chunk(0); - chunkToUse = std::dynamic_pointer_cast(chunkToUse)->values(); - auto array = std::static_pointer_cast(chunkToUse); - REQUIRE(array->length() == 2000); - - const int* ptr = reinterpret_cast(array->values()->data()); - for (size_t i = 0; i < 1000; i++) { - CHECK(ptr[2 * i + 0] == i); - CHECK(ptr[2 * i + 1] == i + 1); - } - } -} - namespace o2::aod { namespace test @@ -149,60 +67,6 @@ DECLARE_SOA_TABLE(Test, "AOD", "ETAPHI", test::Random, test::Ev); } // namespace o2::aod -TEST_CASE("RootTree2TableViaASoA") -{ - using namespace o2::framework; - /// Create a simple TTree - TTree t2("t2", "a simple Tree with simple variables"); - Float_t xyz[3]; - Int_t ij[2]; - Float_t px, py, pz; - Double_t random; - Int_t ev; - t2.Branch("px", &px, "px/F"); - t2.Branch("py", &py, "py/F"); - t2.Branch("pz", &pz, "pz/F"); - t2.Branch("random", &random, "random/D"); - t2.Branch("ev", &ev, "ev/I"); - t2.Branch("xyz", xyz, "xyz[3]/F"); - t2.Branch("ij", ij, "ij[2]/I"); - // fill the tree - for (Int_t i = 0; i < 1000; i++) { - gRandom->Rannor(xyz[0], xyz[1]); - gRandom->Rannor(px, py); - pz = px * px + py * py; - xyz[2] = i + 1; - ij[0] = i; - ij[1] = i + 1; - random = gRandom->Rndm(); - ev = i + 1; - t2.Fill(); - } - - // Create an arrow table from this. - TableBuilder builder; - TTreeReader reader(&t2); - REQUIRE(t2.GetEntries() == 1000); - - RootTableBuilderHelpers::convertASoA(builder, reader); - auto table = builder.finalize(); - REQUIRE(table->num_rows() == 1000); - REQUIRE(table->num_columns() == 7); - REQUIRE(table->column(0)->type()->id() == arrow::float32()->id()); - REQUIRE(table->column(1)->type()->id() == arrow::float32()->id()); - REQUIRE(table->column(2)->type()->id() == arrow::float32()->id()); - REQUIRE(table->column(3)->type()->id() == arrow::fixed_size_list(arrow::float32(), 3)->id()); - REQUIRE(table->column(4)->type()->id() == arrow::fixed_size_list(arrow::int32(), 2)->id()); - REQUIRE(table->column(5)->type()->id() == arrow::float64()->id()); - REQUIRE(table->column(6)->type()->id() == arrow::int32()->id()); - - o2::aod::Test testTable{table}; - for (auto& row : testTable) { - REQUIRE(row.ij()[0] == row.ij()[1] - 1); - REQUIRE(row.ij()[1] == row.ev()); - } -} - TEST_CASE("RootTree2Fragment") { using namespace o2::framework;