diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt index 7202e2299b7cc..f059984b5d85d 100644 --- a/Framework/Core/CMakeLists.txt +++ b/Framework/Core/CMakeLists.txt @@ -250,7 +250,6 @@ add_executable(o2-test-framework-core test/test_Variants.cxx test/test_WorkflowHelpers.cxx test/test_WorkflowSerialization.cxx - test/test_TreeToTable.cxx test/test_DataOutputDirector.cxx test/unittest_SimpleOptionsRetriever.cxx test/unittest_DataSpecUtils.cxx @@ -348,7 +347,6 @@ foreach(b EventMixing HistogramRegistry TableToTree - TreeToTable ExternalFairMQDeviceProxies ) o2_add_executable(benchmark-${b} diff --git a/Framework/Core/include/Framework/DataAllocator.h b/Framework/Core/include/Framework/DataAllocator.h index eb63b5469bb29..287513ec85845 100644 --- a/Framework/Core/include/Framework/DataAllocator.h +++ b/Framework/Core/include/Framework/DataAllocator.h @@ -233,15 +233,6 @@ class DataAllocator return tb; } - template - requires(requires { static_cast(std::declval>()); }) - decltype(auto) make(const Output& spec, Args... args) - { - auto t2t = std::move(LifetimeHolder(new std::decay_t(args...))); - adopt(spec, t2t); - return t2t; - } - template requires(requires { static_cast(std::declval>()); }) decltype(auto) make(const Output& spec, Args... args) @@ -288,11 +279,6 @@ class DataAllocator void adopt(const Output& spec, LifetimeHolder&); - /// Adopt a Tree2Table in the framework and serialise / send - /// it as an Arrow table to all consumers of @a spec once done - void - adopt(const Output& spec, LifetimeHolder&); - /// Adopt a Source2Batch in the framework and serialise / send /// it as an Arrow Dataset to all consumers of @a spec once done void diff --git a/Framework/Core/include/Framework/TableTreeHelpers.h b/Framework/Core/include/Framework/TableTreeHelpers.h index 92725d186ee33..3f76298a5bbd4 100644 --- a/Framework/Core/include/Framework/TableTreeHelpers.h +++ b/Framework/Core/include/Framework/TableTreeHelpers.h @@ -36,19 +36,6 @@ namespace o2::framework // OR t2t.addBranch(column.get(), field.get()), ...; // . t2t.process(); // -// ............................................................................. -// ----------------------------------------------------------------------------- -// TreeToTable allows to fill the contents of a given TTree to an arrow::Table -// ColumnIterator is used by TreeToTable -// -// To copy the contents of a tree tr to a table ta do: -// . TreeToTable t2t(tr); -// . t2t.addColumn(columnname1); t2t.addColumn(columnname2); ... -// OR -// t2t.addAllColumns(); -// . auto ta = t2t.process(); -// -// ............................................................................. struct ROOTTypeInfo { EDataType type; char suffix[3]; @@ -58,29 +45,6 @@ struct ROOTTypeInfo { auto arrowTypeFromROOT(EDataType type, int size); auto basicROOTTypeFromArrow(arrow::Type::type id); -class BranchToColumn -{ - public: - BranchToColumn(TBranch* branch, bool VLA, std::string name, EDataType type, int listSize, arrow::MemoryPool* pool); - // BranchToColumn(TBranch* branch, TBranch* sizeBranch, std::string name, EDataType type, arrow::MemoryPool* pool); - ~BranchToColumn() = default; - TBranch* branch(); - - std::pair, std::shared_ptr> read(TBuffer* buffer); - - private: - TBranch* mBranch = nullptr; - bool mVLA = false; - std::string mColumnName; - EDataType mType; - std::shared_ptr mArrowType; - arrow::ArrayBuilder* mValueBuilder = nullptr; - std::unique_ptr mListBuilder = nullptr; - int mListSize = 1; - std::unique_ptr mBuilder = nullptr; - arrow::MemoryPool* mPool = nullptr; -}; - class ColumnToBranch { public: @@ -127,24 +91,6 @@ class TableToTree std::vector> mColumnReaders; }; -class TreeToTable -{ - public: - TreeToTable(arrow::MemoryPool* pool = arrow::default_memory_pool()); - void setLabel(const char* label); - void addAllColumns(TTree* tree, std::vector&& names = {}); - void fill(TTree*); - std::shared_ptr finalize(); - - private: - arrow::MemoryPool* mArrowMemoryPool; - std::vector> mBranchReaders; - std::string mTableLabel; - std::shared_ptr mTable; - - void addReader(TBranch* branch, std::string const& name, bool VLA); -}; - class FragmentToBatch { public: diff --git a/Framework/Core/src/DataAllocator.cxx b/Framework/Core/src/DataAllocator.cxx index b735eee1f3308..ca35089fdfaab 100644 --- a/Framework/Core/src/DataAllocator.cxx +++ b/Framework/Core/src/DataAllocator.cxx @@ -241,38 +241,6 @@ void DataAllocator::adopt(const Output& spec, LifetimeHolder& tb) context.addBuffer(std::move(header), buffer, std::move(finalizer), routeIndex); } -void DataAllocator::adopt(const Output& spec, LifetimeHolder& t2t) -{ - auto& timingInfo = mRegistry.get(); - RouteIndex routeIndex = matchDataHeader(spec, timingInfo.timeslice); - - auto header = headerMessageFromOutput(spec, routeIndex, o2::header::gSerializationMethodArrow, 0); - auto& context = mRegistry.get(); - - auto creator = [transport = context.proxy().getOutputTransport(routeIndex)](size_t s) -> std::unique_ptr { - return transport->CreateMessage(s); - }; - auto buffer = std::make_shared(creator); - - t2t.callback = [buffer = buffer, transport = context.proxy().getOutputTransport(routeIndex)](TreeToTable& tree) { - // Serialization happens in here, so that we can - // get rid of the intermediate tree 2 table object, saving memory. - auto table = tree.finalize(); - doWriteTable(buffer, table.get()); - // deletion happens in the caller - }; - - /// To finalise this we write the table to the buffer. - /// FIXME: most likely not a great idea. We should probably write to the buffer - /// directly in the TableBuilder, incrementally. - auto finalizer = [](std::shared_ptr b) -> void { - // This is empty because we already serialised the object when - // the LifetimeHolder goes out of scope. - }; - - context.addBuffer(std::move(header), buffer, std::move(finalizer), routeIndex); -} - void DataAllocator::adopt(const Output& spec, LifetimeHolder& f2b) { auto& timingInfo = mRegistry.get(); diff --git a/Framework/Core/src/TableTreeHelpers.cxx b/Framework/Core/src/TableTreeHelpers.cxx index 84d4ff171bc39..92231cb9ce069 100644 --- a/Framework/Core/src/TableTreeHelpers.cxx +++ b/Framework/Core/src/TableTreeHelpers.cxx @@ -102,166 +102,6 @@ auto basicROOTTypeFromArrow(arrow::Type::type id) } } -TBranch* BranchToColumn::branch() -{ - return mBranch; -} - -BranchToColumn::BranchToColumn(TBranch* branch, bool VLA, std::string name, EDataType type, int listSize, arrow::MemoryPool* pool) - : mBranch{branch}, - mVLA{VLA}, - mColumnName{std::move(name)}, - mType{type}, - mArrowType{arrowTypeFromROOT(type, listSize)}, - mListSize{listSize}, - mPool{pool} - -{ - if (mType == EDataType::kBool_t) { - if (mListSize > 1) { - auto status = arrow::MakeBuilder(mPool, mArrowType->field(0)->type(), &mBuilder); - if (!status.ok()) { - throw runtime_error("Cannot create value builder"); - } - mListBuilder = std::make_unique(mPool, std::move(mBuilder), mListSize); - mValueBuilder = static_cast(mListBuilder.get())->value_builder(); - } else { - auto status = arrow::MakeBuilder(mPool, mArrowType, &mBuilder); - if (!status.ok()) { - throw runtime_error("Cannot create builder"); - } - mValueBuilder = mBuilder.get(); - } - } -} - -std::pair, std::shared_ptr> BranchToColumn::read(TBuffer* buffer) -{ - O2_SIGNPOST_ID_FROM_POINTER(sid, tabletree_helpers, buffer); - auto totalEntries = mBranch->GetEntries(); - arrow::Status status; - int readEntries = 0; - buffer->Reset(); - std::shared_ptr array; - - if (mType == EDataType::kBool_t) { - // boolean array special case: we need to use builder to create the bitmap - status = mValueBuilder->Reserve(totalEntries * mListSize); - if (mListSize > 1) { - status &= mListBuilder->Reserve(totalEntries); - } - if (!status.ok()) { - throw runtime_error("Failed to reserve memory for array builder"); - } - while (readEntries < totalEntries) { - auto readLast = mBranch->GetBulkRead().GetBulkEntries(readEntries, *buffer); - readEntries += readLast; - status &= static_cast(mValueBuilder)->AppendValues(reinterpret_cast(buffer->GetCurrent()), readLast * mListSize); - } - if (mListSize > 1) { - status &= static_cast(mListBuilder.get())->AppendValues(readEntries); - } - if (!status.ok()) { - throw runtime_error("Failed to append values to array"); - } - if (mListSize > 1) { - status &= mListBuilder->Finish(&array); - } else { - status &= mValueBuilder->Finish(&array); - } - if (!status.ok()) { - throw runtime_error("Failed to create array"); - } - } else { - // other types: use serialized read to build arrays directly - size_t branchSize = mBranch->GetTotBytes(); - auto&& result = arrow::AllocateResizableBuffer(mBranch->GetTotBytes(), mPool); - O2_SIGNPOST_EVENT_EMIT(tabletree_helpers, sid, "BranchToColumn", "Allocating %ld bytes for %{public}s", branchSize, mBranch->GetName()); - if (!result.ok()) { - throw runtime_error("Cannot allocate values buffer"); - } - std::shared_ptr arrowValuesBuffer = std::move(result).ValueUnsafe(); - auto ptr = arrowValuesBuffer->mutable_data(); - if (ptr == nullptr) { - throw runtime_error("Invalid buffer"); - } - - auto typeSize = TDataType::GetDataType(mType)->Size(); - std::unique_ptr offsetBuffer = nullptr; - - uint32_t offset = 0; - int count = 0; - std::shared_ptr arrowOffsetBuffer; - gsl::span offsets; - int size = 0; - uint32_t totalSize = 0; - TBranch* mSizeBranch = nullptr; - if (mVLA) { - mSizeBranch = mBranch->GetTree()->GetBranch((std::string{mBranch->GetName()} + TableTreeHelpers::sizeBranchSuffix).c_str()); - offsetBuffer = std::make_unique(TBuffer::EMode::kWrite, 4 * 1024 * 1024); - result = arrow::AllocateResizableBuffer((totalEntries + 1) * (int64_t)sizeof(int), mPool); - if (!result.ok()) { - throw runtime_error("Cannot allocate offset buffer"); - } - arrowOffsetBuffer = std::move(result).ValueUnsafe(); - unsigned char* ptrOffset = arrowOffsetBuffer->mutable_data(); - auto* tPtrOffset = reinterpret_cast(ptrOffset); - offsets = gsl::span{tPtrOffset, tPtrOffset + totalEntries + 1}; - - // read sizes first - while (readEntries < totalEntries) { - auto readLast = mSizeBranch->GetBulkRead().GetEntriesSerialized(readEntries, *offsetBuffer); - readEntries += readLast; - for (auto i = 0; i < readLast; ++i) { - offsets[count++] = (int)offset; - offset += swap32_(reinterpret_cast(offsetBuffer->GetCurrent())[i]); - } - } - offsets[count] = (int)offset; - totalSize = offset; - readEntries = 0; - } - - while (readEntries < totalEntries) { - auto readLast = mBranch->GetBulkRead().GetEntriesSerialized(readEntries, *buffer); - if (mVLA) { - size = offsets[readEntries + readLast] - offsets[readEntries]; - } else { - size = readLast * mListSize; - } - readEntries += readLast; - swapCopy(ptr, buffer->GetCurrent(), size, typeSize); - ptr += (ptrdiff_t)(size * typeSize); - } - if (!mVLA) { - totalSize = readEntries * mListSize; - } - std::shared_ptr varray; - switch (mListSize) { - case -1: - varray = std::make_shared(mArrowType->field(0)->type(), totalSize, arrowValuesBuffer); - array = std::make_shared(mArrowType, readEntries, arrowOffsetBuffer, varray); - break; - case 1: - array = std::make_shared(mArrowType, readEntries, arrowValuesBuffer); - break; - default: - varray = std::make_shared(mArrowType->field(0)->type(), totalSize, arrowValuesBuffer); - array = std::make_shared(mArrowType, readEntries, varray); - } - } - - auto fullArray = std::make_shared(array); - auto field = std::make_shared(mBranch->GetName(), mArrowType); - - mBranch->SetStatus(false); - mBranch->DropBaskets("all"); - mBranch->Reset(); - mBranch->GetTransientBuffer(0)->Expand(0); - - return std::make_pair(fullArray, field); -} - ColumnToBranch::ColumnToBranch(TTree* tree, std::shared_ptr const& column, std::shared_ptr const& field) : mBranchName{field->name()}, mColumn{column.get()}, @@ -447,11 +287,6 @@ std::shared_ptr TableToTree::process() return mTree; } -TreeToTable::TreeToTable(arrow::MemoryPool* pool) - : mArrowMemoryPool{pool} -{ -} - namespace { struct BranchInfo { @@ -461,113 +296,6 @@ struct BranchInfo { }; } // namespace -void TreeToTable::addAllColumns(TTree* tree, std::vector&& names) -{ - auto branches = tree->GetListOfBranches(); - auto n = branches->GetEntries(); - if (n == 0) { - throw runtime_error("Tree has no branches"); - } - - std::vector branchInfos; - for (auto i = 0; i < n; ++i) { - auto branch = static_cast(branches->At(i)); - auto name = std::string{branch->GetName()}; - auto pos = name.find(TableTreeHelpers::sizeBranchSuffix); - if (pos != std::string::npos) { - name.erase(pos); - branchInfos.emplace_back(BranchInfo{name, (TBranch*)nullptr, true}); - } else { - auto lookup = std::find_if(branchInfos.begin(), branchInfos.end(), [&](BranchInfo const& bi) { - return bi.name == name; - }); - if (lookup == branchInfos.end()) { - branchInfos.emplace_back(BranchInfo{name, branch, false}); - } else { - lookup->ptr = branch; - } - } - } - - if (names.empty()) { - for (auto& bi : branchInfos) { - addReader(bi.ptr, bi.name, bi.mVLA); - } - } else { - for (auto& name : names) { - auto lookup = std::find_if(branchInfos.begin(), branchInfos.end(), [&](BranchInfo const& bi) { - return name == bi.name; - }); - if (lookup != branchInfos.end()) { - addReader(lookup->ptr, lookup->name, lookup->mVLA); - } - } - if (names.size() != mBranchReaders.size()) { - LOGF(warn, "Not all requested columns were found in the tree"); - } - } - if (mBranchReaders.empty()) { - throw runtime_error("No columns will be read"); - } - // Was affected by https://github.com/root-project/root/issues/8962 - // Re-enabling this seems to cut the number of IOPS in half - tree->SetCacheSize(25000000); - // tree->SetClusterPrefetch(true); - for (auto& reader : mBranchReaders) { - tree->AddBranchToCache(reader->branch()); - if (strncmp(reader->branch()->GetName(), "fIndexArray", strlen("fIndexArray")) == 0) { - std::string sizeBranchName = reader->branch()->GetName(); - sizeBranchName += "_size"; - auto* sizeBranch = (TBranch*)tree->GetBranch(sizeBranchName.c_str()); - if (sizeBranch) { - tree->AddBranchToCache(sizeBranch); - } - } - } - tree->StopCacheLearningPhase(); -} - -void TreeToTable::setLabel(const char* label) -{ - mTableLabel = label; -} - -void TreeToTable::fill(TTree* tree) -{ - std::vector> columns; - std::vector> fields; - static TBufferFile buffer{TBuffer::EMode::kWrite, 4 * 1024 * 1024}; - O2_SIGNPOST_ID_FROM_POINTER(sid, tabletree_helpers, &buffer); - O2_SIGNPOST_START(tabletree_helpers, sid, "TreeToTable", "Filling %{public}s", tree->GetName()); - for (auto& reader : mBranchReaders) { - buffer.Reset(); - auto arrayAndField = reader->read(&buffer); - columns.push_back(arrayAndField.first); - fields.push_back(arrayAndField.second); - } - O2_SIGNPOST_END(tabletree_helpers, sid, "TreeToTable", "Done filling."); - - auto schema = std::make_shared(fields, std::make_shared(std::vector{std::string{"label"}}, std::vector{mTableLabel})); - mTable = arrow::Table::Make(schema, columns); -} - -void TreeToTable::addReader(TBranch* branch, std::string const& name, bool VLA) -{ - static TClass* cls; - EDataType type; - branch->GetExpectedType(cls, type); - auto listSize = -1; - if (!VLA) { - listSize = static_cast(branch->GetListOfLeaves()->At(0))->GetLenStatic(); - } - mBranchReaders.emplace_back(std::make_unique(branch, VLA, name, type, listSize, mArrowMemoryPool)); -} - -std::shared_ptr TreeToTable::finalize() -{ - return mTable; -} - FragmentToBatch::FragmentToBatch(StreamerCreator creator, std::shared_ptr fragment, arrow::MemoryPool* pool) : mFragment{std::move(fragment)}, mArrowMemoryPool{pool}, diff --git a/Framework/Core/test/benchmark_TreeToTable.cxx b/Framework/Core/test/benchmark_TreeToTable.cxx deleted file mode 100644 index 6eca853a90ce4..0000000000000 --- a/Framework/Core/test/benchmark_TreeToTable.cxx +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#include "Framework/CommonDataProcessors.h" -#include "Framework/TableTreeHelpers.h" -#include "Framework/Logger.h" -#include -#include -#include - -#include - -using namespace o2::framework; -using namespace arrow; -using namespace o2::soa; - -namespace test -{ -DECLARE_SOA_COLUMN_FULL(X, x, float, "x"); -DECLARE_SOA_COLUMN_FULL(Y, y, float, "y"); -DECLARE_SOA_COLUMN_FULL(Z, z, float, "z"); -DECLARE_SOA_DYNAMIC_COLUMN(Sum, sum, [](float x, float y) { return x + y; }); -} // namespace test - -#ifdef __APPLE__ -constexpr unsigned int maxrange = 15; -#else -constexpr unsigned int maxrange = 16; -#endif - -static void BM_TreeToTable(benchmark::State& state) -{ - - // initialize a random generator - std::default_random_engine e1(1234567891); - std::uniform_real_distribution rd(0, 1); - std::normal_distribution rf(5., 2.); - std::discrete_distribution rl({10, 20, 30, 30, 5, 5}); - std::discrete_distribution ri({10, 20, 30, 30, 5, 5}); - - // create a table and fill the columns with random numbers - TableBuilder builder; - auto rowWriter = - builder.persist({"a", "b", "c", "d"}); - for (auto i = 0; i < state.range(0); ++i) { - rowWriter(0, rd(e1), rf(e1), rl(e1), ri(e1)); - } - auto table = builder.finalize(); - - // now convert the table to a tree - TFile fout("tree2table.root", "RECREATE"); - TableToTree ta2tr(table, &fout, "tree2table"); - ta2tr.addAllBranches(); - ta2tr.process(); - fout.Close(); - - // read tree and convert to table again - TFile* f = nullptr; - TreeToTable* tr2ta = nullptr; - for (auto _ : state) { - - // Open file and create tree - f = new TFile("tree2table.root", "READ"); - auto tr = (TTree*)f->Get("tree2table"); - - // benchmark TreeToTable - if (tr) { - tr2ta = new TreeToTable; - tr2ta->addAllColumns(tr); - tr2ta->fill(tr); - auto ta = tr2ta->finalize(); - } else { - LOG(info) << "tree is empty!"; - } - - // clean up - delete tr2ta; - - f->Close(); - delete f; - } - - state.SetBytesProcessed(state.iterations() * state.range(0) * 24); -} - -BENCHMARK(BM_TreeToTable)->Range(8, 8 << maxrange); - -BENCHMARK_MAIN(); diff --git a/Framework/Core/test/test_TreeToTable.cxx b/Framework/Core/test/test_TreeToTable.cxx deleted file mode 100644 index 4f3429a5bba62..0000000000000 --- a/Framework/Core/test/test_TreeToTable.cxx +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#include - -#include "Framework/CommonDataProcessors.h" -#include "Framework/TableTreeHelpers.h" -#include "Framework/Logger.h" -#include "Framework/TableBuilder.h" - -#include -#include -#include -#include - -using namespace o2::framework; - -TEST_CASE("TreeToTableConversion") -{ - /// Create a simple TTree - Int_t ndp = 17; - - TFile f1("tree2table.root", "RECREATE"); - TTree t1("t1", "a simple Tree with simple variables"); - Bool_t ok, ts[5] = {false}; - Float_t px, py, pz; - Double_t random; - Int_t ev; - uint8_t b; - const Int_t nelem = 9; - Double_t ij[nelem] = {0}; - float xyzw[96]; - memset(xyzw, 1, 96 * 4); - TString leaflist = Form("ij[%i]/D", nelem); - - Int_t ncols = 10; - t1.Branch("ok", &ok, "ok/O"); - t1.Branch("px", &px, "px/F"); - t1.Branch("py", &py, "py/F"); - t1.Branch("pz", &pz, "pz/F"); - t1.Branch("random", &random, "random/D"); - t1.Branch("ev", &ev, "ev/I"); - t1.Branch("ij", ij, leaflist.Data()); - t1.Branch("tests", ts, "tests[5]/O"); - t1.Branch("xyzw", xyzw, "xyzw[96]/F"); - t1.Branch("small", &b, "small/b"); - - // fill the tree - int ntruein[2] = {0}; - for (int i = 0; i < ndp; i++) { - ok = (i % 2) == 0; - if (ok) { - ntruein[0]++; - } - gRandom->Rannor(px, py); - pz = px * px + py * py; - random = gRandom->Rndm(); - ev = i + 1; - b = i % 3; - for (Int_t jj = 0; jj < nelem; jj++) { - ij[jj] = i + 100 * jj; - } - for (Int_t jj = 0; jj < 5; jj++) { - ts[jj] = (((i + jj) % 2) == 0); - if (ts[jj]) { - ntruein[1]++; - } - } - - t1.Fill(); - } - t1.Write(); - - // Create an arrow table from this. - TreeToTable tr2ta; - tr2ta.addAllColumns(&t1); - tr2ta.fill(&t1); - auto table = tr2ta.finalize(); - f1.Close(); - - // test result - REQUIRE(table->Validate().ok() == true); - REQUIRE(table->num_rows() == ndp); - REQUIRE(table->num_columns() == ncols); - - REQUIRE(table->column(0)->type()->id() == arrow::Type::BOOL); - REQUIRE(table->column(1)->type()->id() == arrow::Type::FLOAT); - REQUIRE(table->column(2)->type()->id() == arrow::Type::FLOAT); - REQUIRE(table->column(3)->type()->id() == arrow::Type::FLOAT); - REQUIRE(table->column(4)->type()->id() == arrow::Type::DOUBLE); - REQUIRE(table->column(5)->type()->id() == arrow::Type::INT32); - REQUIRE(table->column(6)->type()->id() == arrow::Type::FIXED_SIZE_LIST); - REQUIRE(table->column(7)->type()->id() == arrow::Type::FIXED_SIZE_LIST); - REQUIRE(table->column(8)->type()->id() == arrow::Type::FIXED_SIZE_LIST); - REQUIRE(table->column(9)->type()->id() == arrow::Type::UINT8); - - REQUIRE(table->column(0)->type()->Equals(arrow::boolean())); - REQUIRE(table->column(1)->type()->Equals(arrow::float32())); - REQUIRE(table->column(2)->type()->Equals(arrow::float32())); - REQUIRE(table->column(3)->type()->Equals(arrow::float32())); - REQUIRE(table->column(4)->type()->Equals(arrow::float64())); - REQUIRE(table->column(5)->type()->Equals(arrow::int32())); - REQUIRE(table->column(6)->type()->Equals(arrow::fixed_size_list(arrow::float64(), nelem))); - REQUIRE(table->column(7)->type()->Equals(arrow::fixed_size_list(arrow::boolean(), 5))); - REQUIRE(table->column(8)->type()->Equals(arrow::fixed_size_list(arrow::float32(), 96))); - REQUIRE(table->column(9)->type()->Equals(arrow::uint8())); - - // count number of rows with ok==true - int ntrueout = 0; - auto chunks = table->column(0); - REQUIRE(!(chunks.get() == nullptr)); - - auto oks = std::dynamic_pointer_cast(chunks->chunk(0)); - REQUIRE(!(oks.get() == nullptr)); - - for (int ii = 0; ii < table->num_rows(); ii++) { - ntrueout += oks->Value(ii) ? 1 : 0; - } - REQUIRE(ntruein[0] == ntrueout); - - // count number of ts with ts==true - chunks = table->column(7); - REQUIRE(!(chunks.get() == nullptr)); - - auto chunkToUse = std::static_pointer_cast(chunks->chunk(0))->values(); - REQUIRE(!(chunkToUse.get() == nullptr)); - - auto tests = std::dynamic_pointer_cast(chunkToUse); - ntrueout = 0; - for (int ii = 0; ii < table->num_rows() * 5; ii++) { - ntrueout += tests->Value(ii) ? 1 : 0; - } - REQUIRE(ntruein[1] == ntrueout); - - // save table as tree - TFile* f2 = TFile::Open("table2tree.root", "RECREATE"); - TableToTree ta2tr(table, f2, "mytree"); - ta2tr.addAllBranches(); - - auto t2 = ta2tr.process(); - auto br = (TBranch*)t2->GetBranch("ok"); - REQUIRE(t2->GetEntries() == ndp); - REQUIRE(br->GetEntries() == ndp); - br = (TBranch*)t2->GetBranch("tests"); - REQUIRE(br->GetEntries() == ndp); - - f2->Close(); -} - -namespace o2::aod -{ -namespace cols -{ -DECLARE_SOA_COLUMN(Ivec, ivec, std::vector); -DECLARE_SOA_COLUMN(Fvec, fvec, std::vector); -DECLARE_SOA_COLUMN(Dvec, dvec, std::vector); -DECLARE_SOA_COLUMN(UIvec, uivec, std::vector); -} // namespace cols - -DECLARE_SOA_TABLE(Vectors, "AOD", "VECS", o2::soa::Index<>, cols::Ivec, cols::Fvec, cols::Dvec, cols::UIvec); -} // namespace o2::aod - -TEST_CASE("VariableLists") -{ - TableBuilder b; - auto writer = b.cursor(); - std::vector iv; - std::vector fv; - std::vector dv; - std::vector ui; - - std::array empty = {3, 7, 10}; - auto count = 0; - for (auto i = 1; i < 1000; ++i) { - iv.clear(); - fv.clear(); - dv.clear(); - ui.clear(); - if (count < empty.size() && i != empty[count]) { - for (auto j = 0; j < i % 10 + 1; ++j) { - iv.push_back(j + 2); - fv.push_back((j + 2) * 0.2134f); - dv.push_back((j + 4) * 0.192873819237); - ui.push_back(j); - } - } else { - count++; - } - writer(0, iv, fv, dv, ui); - } - auto table = b.finalize(); - - auto* f = TFile::Open("variable_lists.root", "RECREATE"); - TableToTree ta2tr(table, f, "lists"); - ta2tr.addAllBranches(); - auto tree = ta2tr.process(); - f->Close(); - - auto* f2 = TFile::Open("variable_lists.root", "READ"); - auto* treeptr = static_cast(f2->Get("lists;1")); - TreeToTable tr2ta; - tr2ta.addAllColumns(treeptr); - tr2ta.fill(treeptr); - auto ta = tr2ta.finalize(); - o2::aod::Vectors v{ta}; - int i = 1; - count = 0; - for (auto& row : v) { - auto ivr = row.ivec(); - auto fvr = row.fvec(); - auto dvr = row.dvec(); - auto uvr = row.uivec(); - if (count < empty.size() && i != empty[count]) { - for (auto j = 0; j < i % 10 + 1; ++j) { - REQUIRE(ivr[j] == j + 2); - REQUIRE(fvr[j] == (j + 2) * 0.2134f); - REQUIRE(dvr[j] == (j + 4) * 0.192873819237); - REQUIRE(uvr[j] == j); - } - } else { - REQUIRE(ivr.size() == 0); - REQUIRE(fvr.size() == 0); - REQUIRE(dvr.size() == 0); - REQUIRE(uvr.size() == 0); - count++; - } - ++i; - } -}