From eebb9e566ba6ab949abac600d9c826118c4b1bd8 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 4 Sep 2025 15:52:07 +0200 Subject: [PATCH 01/25] GPU TPC: Remove option to retry refit if cluster rejection breaks the track, we will anyway rebuild the track in the future --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 - .../Global/GPUChainTrackingMerger.cxx | 3 -- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 3 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 3 -- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 7 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 49 +++++-------------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 4 +- 7 files changed, 17 insertions(+), 53 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index d70fac115eab7..5e4fc73a11f28 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -155,7 +155,6 @@ AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation inst AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF") AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF of track for non-interpolated reject (both for chi2 and absolute distance)") AddOptionRTC(mergeCE, uint8_t, 1, "", 0, "Merge tracks accross the central electrode") -AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without cluster rejection when fit fails (=2 means retry in same kernel, =1 for separate kernel") AddOptionRTC(enablePID, int8_t, 1, "", 0, "Enable PID response") AddOptionRTC(PID_useNsigma, int8_t, 1, "", 0, "Use nSigma instead of absolute distance in PID response") AddOptionRTC(adddEdxSubThresholdClusters, int8_t, 1, "", 0, "Add sub threshold clusters in TPC dEdx computation") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 5ab4b963d7330..5d988828e9e74 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -227,9 +227,6 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0); - if (param().rec.tpc.retryRefit == 1) { - runKernel(GetGridAuto(0), -1); - } runKernel(GetGridAuto(0)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRefit, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 260781c17406b..756bbeceddc86 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -402,8 +402,7 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mSharedCount, mNMaxClusters); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers - computePointerWithAlignment(mem, mRetryRefitIds, mNMaxTracks); // Reducing mNMaxTracks for mLoopData / mRetryRefitIds does not save memory, since the other parts are larger anyway + computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 14974bdec2303..ea94204fc789b 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -66,7 +66,6 @@ class GPUTPCGMMerger : public GPUProcessor static constexpr const int32_t NSECTORS = GPUCA_NSECTORS; //* N sectors struct memory { - GPUAtomic(uint32_t) nRetryRefit; GPUAtomic(uint32_t) nLoopData; GPUAtomic(uint32_t) nUnpackedTracks; GPUAtomic(uint32_t) nMergedTracks; @@ -119,7 +118,6 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } - GPUhdi() uint32_t* RetryRefitIds() const { return mRetryRefitIds; } GPUhdi() uint8_t* ClusterStateExt() const { return mClusterStateExt; } GPUhdi() GPUTPCGMLoopData* LoopData() const { return mLoopData; } GPUhdi() memory* Memory() const { return mMemory; } @@ -287,7 +285,6 @@ class GPUTPCGMMerger : public GPUProcessor gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRangeMemory = nullptr; // memory for border tracks gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRange[NSECTORS]; // memory for border tracks memory* mMemory = nullptr; - uint32_t* mRetryRefitIds = nullptr; GPUTPCGMLoopData* mLoopData = nullptr; }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 3cb937d4f7abc..28d0767a264d9 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -21,10 +21,9 @@ using namespace o2::gpu; template <> GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode) { - const int32_t iEnd = mode == -1 ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); - GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, iEnd, { - const int32_t i = mode == -1 ? merger.RetryRefitIds()[ii] : mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, mode == -1); + GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { + const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 70fb9cd1a769e..89bcb50fdcd43 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -48,7 +48,7 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, int32_t attempt, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) { static constexpr float kDeg2Rad = M_PI / 180.f; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); @@ -92,7 +92,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); ResetCovariance(); - prop.SetSeedingErrors(!(refit && attempt == 0)); + prop.SetSeedingErrors(!(refit)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger->GetConstantMem()->calibObjects.matLUT : nullptr); @@ -105,7 +105,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - int32_t goodRows = 0; for (int32_t ihit = ihitStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); if (crossCE) { @@ -179,11 +178,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); } } - if (lastRow == 255 || CAMath::Abs((int32_t)lastRow - (int32_t)cluster.row) > 5 || lastSector != cluster.sector || (param.rec.tpc.trackFitRejectMode < 0 && -nMissed <= param.rec.tpc.trackFitRejectMode)) { - goodRows = 0; - } else { - goodRows++; - } if (retValProp == 0) { lastRow = cluster.row; lastSector = cluster.sector; @@ -228,17 +222,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; } else { int8_t rejectChi2 = 0; - if (attempt == 0) { - if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { - if (iWay == nWays - 3) { - rejectChi2 = GPUTPCGMPropagator::rejectInterFill; - } else if (iWay == nWays - 2) { - rejectChi2 = GPUTPCGMPropagator::rejectInterReject; - } else if (iWay == nWays - 1) { - rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; - } - } else { - rejectChi2 = allowChangeClusters && goodRows > 5; + if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { + if (iWay == nWays - 3) { + rejectChi2 = GPUTPCGMPropagator::rejectInterFill; + } else if (iWay == nWays - 2) { + rejectChi2 = GPUTPCGMPropagator::rejectInterReject; + } else if (iWay == nWays - 1) { + rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; } } @@ -905,7 +895,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger, int32_t attempt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. { if (!track.OK()) { return; @@ -920,27 +910,10 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); CADEBUG(int32_t nTrackHitsOld = nTrackHits; float ptOld = t.QPt()); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, attempt, GPUCA_MAX_SIN_PHI, track); + bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, GPUCA_MAX_SIN_PHI, track); CADEBUG(printf("Finished Fit Track %d\n", iTrk)); CADEBUG(printf("OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", nTrackHitsOld, nTrackHits, NTolerated, nTrackHits + NTolerated, ptOld, t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); - if (!ok && attempt == 0 && merger->Param().rec.tpc.retryRefit) { - for (uint32_t i = 0; i < track.NClusters(); i++) { - merger->Clusters()[track.FirstClusterRef() + i].state &= GPUTPCGMMergedTrackHit::clustererAndSharedFlags; - } - CADEBUG(printf("Track rejected, marking for retry\n")); - if (merger->Param().rec.tpc.retryRefit == 2) { - nTrackHits = track.NClusters(); - NTolerated = 0; // Clusters not fit but tollerated for track length cut - t = track.Param(); - Alpha = track.Alpha(); - ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, 1, GPUCA_MAX_SIN_PHI, track); - } else { - uint32_t nRefit = CAMath::AtomicAdd(&merger->Memory()->nRetryRefit, 1u); - merger->RetryRefitIds()[nRefit] = iTrk; - return; - } - } if (CAMath::Abs(t.QPt()) < 1.e-4f) { t.QPt() = 1.e-4f; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index f38ea4d320c14..e2a3245cabfd7 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -141,7 +141,7 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, int32_t attempt, float maxSinPhi, GPUTPCGMMergedTrack& track); + GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); @@ -200,7 +200,7 @@ class GPUTPCGMTrackParam } } - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, int32_t attempt); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger); GPUdi() void ConstrainSinPhi(float limit = GPUCA_MAX_SIN_PHI) { From 0d00fa610457ec574aac646bc4934ae9303caaa4 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 4 Sep 2025 21:34:57 +0200 Subject: [PATCH 02/25] GPU TPC: Add possibility to run last way of TPC fit in separate kernel to rebuild track ion between --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Global/GPUChainTracking.cxx | 4 ++ .../Global/GPUChainTrackingMerger.cxx | 5 ++- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 4 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 44 ++++++++++++------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 5 +-- GPU/GPUTracking/kernels.cmake | 2 +- 8 files changed, 42 insertions(+), 25 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 5e4fc73a11f28..99a6358dca226 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -138,6 +138,7 @@ AddOptionRTC(cfNoiseSuppressionEpsilon, uint8_t, 10, "", 0, "Cluster Finder: Dif AddOptionRTC(cfNoiseSuppressionEpsilonRelative, uint8_t, 76, "", 0, "Cluster Finder: Difference between peak and charge for the charge to count as a minima during noise suppression, relative as fraction of 255") AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 pads closes to the sector edge as edge cluster") AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") +AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index f370b756e2cdb..2925bfefc2008 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -257,6 +257,10 @@ bool GPUChainTracking::ValidateSettings() GPUError("Cannot do error interpolation with NWays < 3!"); return false; } + if (param().rec.tpc.rebuildTrackInFit && !param().rec.tpc.mergerInterpolateErrors) { + GPUError("Need error interpolation to rebuild tracks during fit"); + return false; + } if (param().continuousMaxTimeBin > (int32_t)GPUSettings::TPC_MAX_TF_TIME_BIN) { GPUError("configured max time bin exceeds 256 orbits"); return false; diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 5d988828e9e74..1c3a009de5233 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -226,7 +226,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) mOutputQueue.clear(); } - runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); + if (param().rec.tpc.rebuildTrackInFit) { + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); + } runKernel(GetGridAuto(0)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRefit, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 28d0767a264d9..0b2229a33d0fc 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -19,11 +19,11 @@ using namespace o2::gpu; template <> -GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode) +GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode, int32_t rebuilt) { GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger); + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, rebuilt); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index 5d00451516aa8..0e3e4990ff04b 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -36,7 +36,7 @@ class GPUTPCGMMergerTrackFit : public GPUTPCGMMergerGeneral { public: template - GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t mode); + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t mode, int32_t rebuilt); }; class GPUTPCGMMergerFollowLoopers : public GPUTPCGMMergerGeneral diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 89bcb50fdcd43..ed4aae273fc84 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -48,7 +48,7 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { static constexpr float kDeg2Rad = M_PI / 180.f; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); @@ -61,7 +61,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ prop.SetMaterialTPC(); prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); - if (param.rec.tpc.mergerInterpolateErrors) { + if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { for (int32_t i = 0; i < N; i++) { interpolation.hit[i].errorY = -1; } @@ -71,12 +71,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const int32_t maxN = N; int32_t ihitStart = 0; float covYYUpd = 0.f; - float lastUpdateX = -1.f; - uint8_t lastRow = 255; - uint8_t lastSector = 255; float deltaZ = 0.f; - for (int32_t iWay = 0; iWay < nWays; iWay++) { + for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissed2 = 0; float sumInvSqrtCharge = 0.f; int32_t nAvgCharge = 0; @@ -96,12 +93,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger->GetConstantMem()->calibObjects.matLUT : nullptr); - prop.SetTrack(this, iWay ? prop.GetAlpha() : Alpha); + prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : GPUCA_MAX_SIN_PHI_LOW); CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / kSectAngle) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = 0; - lastUpdateX = -1; + uint8_t lastRow = 255; + uint8_t lastSector = 255; + float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; @@ -111,9 +110,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ lastSector = clusters[ihit].sector; } - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) { + if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); - if (finalOutInFit && !(clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject)) { + if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { + NTolerated++; + } + if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; } continue; @@ -328,6 +330,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } else { deltaZ = 0.f; } + + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + Alpha = prop.GetAlpha(); + if (ihitStart != 0) { + MarkClusters(clusters, 0, ihitStart - 1, 1, GPUTPCGMMergedTrackHit::flagHighIncl); + } + return true; + } } ConstrainSinPhi(); @@ -895,7 +905,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. { if (!track.OK()) { return; @@ -909,21 +919,21 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - CADEBUG(int32_t nTrackHitsOld = nTrackHits; float ptOld = t.QPt()); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, GPUCA_MAX_SIN_PHI, track); - CADEBUG(printf("Finished Fit Track %d\n", iTrk)); - CADEBUG(printf("OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", nTrackHitsOld, nTrackHits, NTolerated, nTrackHits + NTolerated, ptOld, t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); + bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, GPUCA_MAX_SIN_PHI, track, rebuilt); + CADEBUG(if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (CAMath::Abs(t.QPt()) < 1.e-4f) { - t.QPt() = 1.e-4f; + t.QPt() = CAMath::Copysign(1.e-4f, t.QPt()); } CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger->MergedTracks()[iTrk].Looper()); }); track.SetOK(ok); - track.SetNClustersFitted(nTrackHits); track.Param() = t; track.Alpha() = Alpha; + if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) { + track.SetNClustersFitted(nTrackHits); + } // if (track.OK()) merger->DebugRefitMergedTrack(track); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index e2a3245cabfd7..ce7c9247b8c78 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -141,7 +141,8 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track); + GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); @@ -200,8 +201,6 @@ class GPUTPCGMTrackParam } } - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger); - GPUdi() void ConstrainSinPhi(float limit = GPUCA_MAX_SIN_PHI) { if (mP[2] > limit) { diff --git a/GPU/GPUTracking/kernels.cmake b/GPU/GPUTracking/kernels.cmake index 4d1abe9f5b064..8d8d2102c3b6c 100644 --- a/GPU/GPUTracking/kernels.cmake +++ b/GPU/GPUTracking/kernels.cmake @@ -51,7 +51,7 @@ o2_gpu_add_kernel("GPUTPCGlobalDebugSortKernels, mergedTracks2" "= TPC o2_gpu_add_kernel("GPUTPCGlobalDebugSortKernels, borderTracks" "= TPCMERGER" NO int8_t parameter) o2_gpu_add_kernel("GPUTPCCreateOccupancyMap, fill" "= TPCOCCUPANCY" LB GPUTPCClusterOccupancyMapBin* map) o2_gpu_add_kernel("GPUTPCCreateOccupancyMap, fold" "= TPCOCCUPANCY" LB GPUTPCClusterOccupancyMapBin* map uint32_t* output) -o2_gpu_add_kernel("GPUTPCGMMergerTrackFit" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB int32_t mode) +o2_gpu_add_kernel("GPUTPCGMMergerTrackFit" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB int32_t mode int32_t rebuilt) o2_gpu_add_kernel("GPUTPCGMMergerFollowLoopers" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT" LB) o2_gpu_add_kernel("GPUTPCGMMergerUnpackResetIds" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iSector) o2_gpu_add_kernel("GPUTPCGMMergerSectorRefit" "GPUTPCGMMergerGPU TPCMERGER MATLUT" LB int32_t iSector) From 258b520f038eb40a28965be0189688648916d2e4 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 23 Sep 2025 17:58:33 +0200 Subject: [PATCH 03/25] GPU TPC: Search hit closest to interpolated point in track rebuilding --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + .../Global/GPUChainTrackingMerger.cxx | 26 ++-- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 3 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 5 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 131 ++++++++++++++++-- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- GPU/GPUTracking/SectorTracker/GPUTPCTracker.h | 1 + .../GPUTPCTrackletConstructor.cxx | 4 +- 8 files changed, 148 insertions(+), 25 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 99a6358dca226..8054f9427051d 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -139,6 +139,7 @@ AddOptionRTC(cfNoiseSuppressionEpsilonRelative, uint8_t, 76, "", 0, "Cluster Fin AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 pads closes to the sector edge as edge cluster") AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") +AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 1c3a009de5233..96053356d69c8 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -94,7 +94,6 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) uint32_t numBlocks = (!mRec->IsGPU() || doGPU) ? BlockCount() : 1; GPUTPCGMMerger& Merger = processors()->tpcMerger; GPUTPCGMMerger& MergerShadow = doGPU ? processorsShadow()->tpcMerger : Merger; - GPUTPCGMMerger& MergerShadowAll = doGPU ? processorsShadow()->tpcMerger : Merger; const int32_t outputStream = OutputStream(); if (GetProcessingSettings().debugLevel >= 2) { GPUInfo("Running TPC Merger"); @@ -139,28 +138,28 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingSectorTracks, doGPU, Merger, &GPUTPCGMMerger::DumpSectorTracks, *mDebugFile); runKernel(GetGridAuto(0, deviceType), false); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridAuto(0, deviceType)); RunTPCTrackingMerger_MergeBorderTracks(1, 0, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingMatching, doGPU, Merger, &GPUTPCGMMerger::DumpMergedWithinSectors, *mDebugFile); runKernel(GetGridAuto(0, deviceType), false); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), 2, 3, 0); RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), 0, 1, 0); RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), 0, 1, 1); RunTPCTrackingMerger_MergeBorderTracks(0, -1, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingMatching, doGPU, Merger, &GPUTPCGMMerger::DumpMergedBetweenSectors, *mDebugFile); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridAuto(0, deviceType)); if (GetProcessingSettings().mergerSanityCheck) { @@ -200,8 +199,8 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) if (maxId > Merger.NMaxClusters()) { throw std::runtime_error("mNMaxClusters too small"); } - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.SharedCount(), maxId * sizeof(*MergerShadowAll.SharedCount())); - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.ClusterAttachment(), maxId * sizeof(*MergerShadowAll.ClusterAttachment())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), maxId * sizeof(*MergerShadow.SharedCount())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), maxId * sizeof(*MergerShadow.ClusterAttachment())); runKernel(GetGridAuto(0, deviceType)); CondWaitEvent(waitForTransfer, &mEvents->single); runKernel(GetGridAuto(0, deviceType)); @@ -226,6 +225,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) mOutputQueue.clear(); } + if (param().rec.tpc.rebuildTrackInFit) { + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), Merger.NMergedTracks() * GPUCA_ROW_COUNT * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); + } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); if (param().rec.tpc.rebuildTrackInFit) { runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); @@ -260,13 +262,13 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) throw std::runtime_error("QA Scratch buffer exceeded"); } } - GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracks(), MergerShadowAll.MergedTracks(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracks()), outputStream, 0, nullptr, waitEvent); + GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracks(), MergerShadow.MergedTracks(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracks()), outputStream, 0, nullptr, waitEvent); waitEvent = nullptr; if (param().dodEdxEnabled) { - GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracksdEdx(), MergerShadowAll.MergedTracksdEdx(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracksdEdx()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracksdEdx(), MergerShadow.MergedTracksdEdx(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracksdEdx()), outputStream, 0); } - GPUMemCpy(RecoStep::TPCMerging, Merger.Clusters(), MergerShadowAll.Clusters(), Merger.NMergedTrackClusters() * sizeof(*Merger.Clusters()), outputStream, 0); - GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadowAll.ClusterAttachment(), Merger.NMaxClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.Clusters(), MergerShadow.Clusters(), Merger.NMergedTrackClusters() * sizeof(*Merger.Clusters()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NMaxClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); } if (GetProcessingSettings().outputSharedClusterMap) { TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResOutputState(), outputStream, nullptr, waitEvent); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 756bbeceddc86..d411d23115a32 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -403,6 +403,7 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway + computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUCA_ROW_COUNT * Param().rec.tpc.rebuildTrackInFitClusterCandidates); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 @@ -1666,7 +1667,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread const GPUTPCTracker& trk = GetConstantMem()->tpcTrackers[t->Sector()]; const GPUTPCHitId& ic = trk.TrackHits()[t->OrigTrack()->FirstHitID() + i]; uint32_t id = trk.Data().ClusterDataIndex(trk.Data().Row(ic.RowIndex()), ic.HitIndex()) + GetConstantMem()->ioPtrs.clustersNative->clusterOffset[t->Sector()][0]; - *c2 = trackCluster{id, (uint8_t)ic.RowIndex(), t->Sector()}; + *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f}; } nHits += nTrackHits; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index ea94204fc789b..f1e0dd0590c90 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -77,10 +77,11 @@ class GPUTPCGMMerger : public GPUProcessor GPUAtomic(uint32_t) nLooperMatchCandidates; }; - struct trackCluster { + struct trackCluster { // TODO: Reduce size of this struct! uint32_t id; uint8_t row; uint8_t sector; + float error; }; struct tmpSort { @@ -115,6 +116,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() uint32_t NMergedTrackClusters() const { return mMemory->nMergedTrackClusters; } GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return (mClusters); } + GPUhdi() trackCluster* ClusterCandidates() { return (mClusterCandidates); } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } @@ -260,6 +262,7 @@ class GPUTPCGMMerger : public GPUProcessor int32_t mNClusters = 0; // Total number of incoming clusters (from sector tracks) GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks + trackCluster* mClusterCandidates = nullptr; GPUdEdxInfo* mMergedTracksdEdx = nullptr; //* dEdx information GPUdEdxInfo* mMergedTracksdEdxAlt = nullptr; //* dEdx alternative information GPUTPCGMSectorTrack* mSectorTrackInfos = nullptr; //* additional information for sector tracks diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index ed4aae273fc84..57c26c6d26db8 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -48,9 +48,10 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { static constexpr float kDeg2Rad = M_PI / 180.f; + static constexpr float maxSinPhi = GPUCA_MAX_SIN_PHI; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); const GPUParam& GPUrestrict() param = merger->Param(); @@ -73,9 +74,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ float covYYUpd = 0.f; float deltaZ = 0.f; - for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // DR: Unrolling has no performance improvement on GPU, why? + for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissed2 = 0; - float sumInvSqrtCharge = 0.f; + float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! int32_t nAvgCharge = 0; if (iWay && (iWay & 1) == 0) { @@ -136,7 +137,26 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ // CADEBUG(if ((uint32_t)merger->GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger->GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); // clang-format on - if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, allowChangeClusters) == -1) { + if (rebuilt && merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id > 0) { + while (true) { + if (!(ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector)) { + break; + } + } + const auto& interHit = merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + if (interHit.id == 1) { + nMissed++; + nMissed2++; + continue; + } + const ClusterNative& GPUrestrict() cl = merger -> GetConstantMem()->ioPtrs.clustersNative->clustersLinear[interHit.id - 2]; + merger->GetConstantMem()->calibObjects.fastTransformHelper->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + if (interHit.id - 2 == clusters[ihit].num) { + clusterState = clusters[ihit].state; + } else { + clusterState = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + } + } else if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; nMissed2++; continue; @@ -156,7 +176,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ // clang-format off CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); // clang-format on - if (allowChangeClusters && lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { + if (!param.rec.tpc.rebuildTrackInFit && allowChangeClusters && lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; dodEdx = AttachClustersPropagate(merger, cluster.sector, lastRow, cluster.row, iTrk, track.Leg() == 0, prop, inFlyDirection, GPUCA_MAX_SIN_PHI, dodEdx); @@ -201,7 +221,102 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } float uncorrectedY = -1e6f; - if (allowChangeClusters) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + const GPUTPCTracker& GPUrestrict() tracker = *(merger -> GetConstantMem()->tpcTrackers + cluster.sector); + const GPUTPCRow& GPUrestrict() row = tracker.Row(cluster.row); + GPUglobalref() const cahit2* hits = tracker.HitData(row); + GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); + const auto& inter = interpolation.hit[ihit]; + if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float zOffset = merger->GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin); + const float y0 = row.Grid().YMin(); + const float stepY = row.HstepY(); + const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway + const float stepZ = row.HstepZ(); + int32_t bin, ny, nz; + + float err2Y, err2Z; + param.GetClusterErrors2(cluster.sector, cluster.row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + + const float Iz0 = inter.posY - mP[0]; + const float Iz1 = inter.posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0; + const float ImP1 = mP[1] + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0]; + const float ImC2 = mC[2] - Ik11 * mC[2]; + + float uncorrectedZ; + merger->GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, ImP0, ImP1, uncorrectedY, uncorrectedZ); + + int32_t nCandidates = 0; + if (CAMath::Abs(uncorrectedY) <= row.getTPCMaxY()) { + const float kFactor = tracker.GetChiSeedFactor(); + const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? + const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); + const float tubeY = CAMath::Sqrt(sy2); + const float tubeZ = CAMath::Sqrt(sz2); + row.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); + + const int32_t nBinsY = row.Grid().Ny(); + const int32_t idOffset = tracker.Data().ClusterIdOffset(); + const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); + for (int32_t k = 0; k <= nz; k++) { + const int32_t mybin = bin + k * nBinsY; + const uint32_t hitFst = firsthit[mybin]; + const uint32_t hitLst = firsthit[mybin + ny + 1]; + for (uint32_t ih = hitFst; ih < hitLst; ih++) { + const cahit2 hh = hits[ih]; + const float y = y0 + hh.x * stepY; + const float z = z0 + hh.y * stepZ; + const float dy = y - uncorrectedY; + const float dz = z - uncorrectedZ; + + if (dy * dy < sy2 && dz * dz < sz2) { + float err2YA, err2ZA; + const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[idOffset + ids[ih]]; + const auto clflags = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + const float time = cl.getTime(); + const float invSqrtCharge = CAMath::InvSqrt(cl.qMax); + const float invCharge = 1.f / cl.qMax; + float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); + invAvgCharge *= invAvgCharge; + + prop.GetErr2(err2YA, err2ZA, merger->Param(), mP[1], cluster.row, clflags, cluster.sector, time, invAvgCharge, invCharge); + const float Jw0 = 1.f / (ImC0 + err2YA); + const float Jw2 = 1.f / (ImC2 + err2ZA); + const float chi2Y = Jw0 * dy * dy; + const float chi2Z = Jw2 * dz * dz; + bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); + float err = dy * dy + dz * dz; + if (ok) { + int32_t insert = nCandidates; + for (int32_t c = 0; c < nCandidates; c++) { + if (err < merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { + insert = c; + break; + } + } + if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { + for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { + merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; + } + merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err}; + nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); + } + } + } + } + } + } + if (nCandidates == 0) { + merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; + } + } + } else if (allowChangeClusters) { uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } @@ -321,7 +436,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ break; // bad chi2 for the whole track, stop the fit } } - if (finalOutInFit && !(merger->Param().rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastRow %d row %d out %d\n", iTrk, (int)lastRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -919,7 +1034,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, GPUCA_MAX_SIN_PHI, track, rebuilt); + bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, track, rebuilt); CADEBUG(if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (CAMath::Abs(t.QPt()) < 1.e-4f) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index ce7c9247b8c78..9f2d1b97c6981 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -141,7 +141,7 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h index 5efd3ca845410..66216181fdee6 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h @@ -167,6 +167,7 @@ class GPUTPCTracker : public GPUProcessor } return ((int32_t)weight); } + GPUdi() float GetChiSeedFactor() const { return Param().rec.tpc.hitPickUpFactor * Param().rec.tpc.hitPickUpFactor * 3.5f * 3.5f; } GPUd() void MaximizeHitWeight(const GPUTPCRow& row, int32_t hitIndex, int32_t weight) { mData.MaximizeHitWeight(row, hitIndex, weight); } GPUd() void SetHitWeight(const GPUTPCRow& row, int32_t hitIndex, int32_t weight) { mData.SetHitWeight(row, hitIndex, weight); } GPUd() int32_t HitWeight(const GPUTPCRow& row, int32_t hitIndex) const { return mData.HitWeight(row, hitIndex); } diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx index 567e4e93cc524..c1dad52ec61d9 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx @@ -199,7 +199,7 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int32_t /*nBlocks*/, const float sErr2 = tracker.Param().GetSystematicClusterErrorIFC2(x, tParam.GetY(), tParam.GetZ(), tracker.ISector() >= 18); err2Y += sErr2; err2Z += sErr2; - const float kFactor = tracker.Param().rec.tpc.hitPickUpFactor * tracker.Param().rec.tpc.hitPickUpFactor * 3.5f * 3.5f; + const float kFactor = tracker.GetChiSeedFactor(); float sy2 = kFactor * (tParam.Err2Y() + err2Y); float sz2 = kFactor * (tParam.Err2Z() + err2Z); if (sy2 > tracker.Param().rec.tpc.hitSearchArea2) { @@ -307,7 +307,7 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int32_t /*nBlocks*/, const float kFactor = tracker.Param().rec.tpc.hitPickUpFactor * tracker.Param().rec.tpc.hitPickUpFactor * 7.0f * 7.0f; const float maxWindow2 = tracker.Param().rec.tpc.hitSearchArea2; const float sy2 = CAMath::Min(maxWindow2, kFactor * (tParam.Err2Y() + err2Y)); - const float sz2 = CAMath::Min(maxWindow2, kFactor * (tParam.Err2Z() + err2Z)); + const float sz2 = CAMath::Min(maxWindow2, kFactor * (tParam.Err2Z() + err2Z)); // TODO: Double check if the factors are enough, errors could be larger with flags int32_t bin, ny, nz; row.Grid().GetBinArea(yUncorrected, zUncorrected + tParam.ZOffset(), CAMath::Sqrt(sy2), CAMath::Sqrt(sz2), bin, ny, nz); From f9ee89832498b172fb233bd138c9b88deda7b4cf Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 5 Oct 2025 18:31:00 +0200 Subject: [PATCH 04/25] GPU TPC: Fix index for hit interpolation in rows with multiple clusters --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 57c26c6d26db8..e92610918c3f4 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -220,13 +220,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } } + auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { const GPUTPCTracker& GPUrestrict() tracker = *(merger -> GetConstantMem()->tpcTrackers + cluster.sector); const GPUTPCRow& GPUrestrict() row = tracker.Row(cluster.row); GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); - const auto& inter = interpolation.hit[ihit]; if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { const float zOffset = merger->GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin); const float y0 = row.Grid().YMin(); @@ -358,10 +358,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { - if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && interpolation.hit[ihit].errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { rejectChi2 = GPUTPCGMPropagator::rejectDirect; } else { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &interpolation.hit[ihit], err2Y, err2Z, deltaZ); + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); } } @@ -371,7 +371,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); } GPUCA_DEBUG_STREAMER_CHECK(if (o2::utils::DebugStreamer::checkStream(o2::utils::StreamFlags::streamUpdateTrack, iTrk)) { - merger->DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, interpolation.hit[ihit], rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); + merger->DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, inter, rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); }); } // clang-format off From 37e98892e10188a17a9f86a536adfb4b9ed65c08 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 29 Sep 2025 22:01:44 +0200 Subject: [PATCH 05/25] GPU TPC: Implement ambiguity solving for rebuild tracks cluster association --- .../Definitions/GPUDefParametersDefaults.h | 30 +++ GPU/GPUTracking/Definitions/GPUSettingsList.h | 3 +- .../GPUChainTrackingDebugAndProfiling.cxx | 6 +- .../Global/GPUChainTrackingMerger.cxx | 22 +- GPU/GPUTracking/Global/GPUErrorCodes.h | 20 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 216 ++++++++++++++++-- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 27 ++- GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 6 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 31 +++ GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h | 13 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 36 +-- GPU/GPUTracking/SectorTracker/GPUTPCTracker.h | 5 +- .../Standalone/Benchmark/standalone.cxx | 2 +- GPU/GPUTracking/kernels.cmake | 5 + GPU/GPUTracking/qa/GPUQA.cxx | 2 +- 15 files changed, 343 insertions(+), 81 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUDefParametersDefaults.h b/GPU/GPUTracking/Definitions/GPUDefParametersDefaults.h index 01ae33dc3b4d8..f8a9d909f556c 100644 --- a/GPU/GPUTracking/Definitions/GPUDefParametersDefaults.h +++ b/GPU/GPUTracking/Definitions/GPUDefParametersDefaults.h @@ -64,6 +64,11 @@ #define GPUCA_LB_GPUTPCGMMergerFinalize_0 256 #define GPUCA_LB_GPUTPCGMMergerFinalize_1 256 #define GPUCA_LB_GPUTPCGMMergerFinalize_2 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_prepare 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_compute 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve1 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve2 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolveShared 256 #define GPUCA_LB_GPUTPCCompressionKernels_step0attached 128, 1 #define GPUCA_LB_GPUTPCCompressionKernels_step1unattached 512, 2 #define GPUCA_LB_GPUTPCDecompressionKernels_step0attached 128, 2 @@ -193,6 +198,11 @@ #define GPUCA_LB_GPUTPCGMMergerFinalize_0 256 #define GPUCA_LB_GPUTPCGMMergerFinalize_1 256 #define GPUCA_LB_GPUTPCGMMergerFinalize_2 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_prepare 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_compute 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve1 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve2 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolveShared 256 #define GPUCA_LB_GPUTPCCompressionKernels_step0attached 64, 2 #define GPUCA_LB_GPUTPCCompressionKernels_step1unattached 512, 3 #define GPUCA_LB_GPUTPCDecompressionKernels_step0attached 32, 1 @@ -255,6 +265,11 @@ #define GPUCA_LB_GPUTPCGMMergerFinalize_0 256 #define GPUCA_LB_GPUTPCGMMergerFinalize_1 256 #define GPUCA_LB_GPUTPCGMMergerFinalize_2 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_prepare 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_compute 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve1 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve2 256 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolveShared 256 #define GPUCA_LB_GPUTPCCompressionKernels_step0attached 128 #define GPUCA_LB_GPUTPCCompressionKernels_step1unattached 512, 2 #define GPUCA_LB_GPUTPCDecompressionKernels_step0attached 32, 1 @@ -425,6 +440,21 @@ #ifndef GPUCA_LB_GPUTPCGMMergerFinalize_step2 #define GPUCA_LB_GPUTPCGMMergerFinalize_step2 256 #endif + #ifndef GPUCA_LB_GPUTPCGMMergerHitWeights_prepare + #define GPUCA_LB_GPUTPCGMMergerHitWeights_prepare 256 + #endif + #ifndef GPUCA_LB_GPUTPCGMMergerHitWeights_compute + #define GPUCA_LB_GPUTPCGMMergerHitWeights_compute 256 + #endif + #ifndef GPUCA_LB_GPUTPCGMMergerHitWeights_resolve1 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve1 256 + #endif + #ifndef GPUCA_LB_GPUTPCGMMergerHitWeights_resolve2 + #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolve2 256 + #endif + #ifndef GPUCA_LB_GPUTPCGMMergerHitWeights_resolveShared + #define GPUCA_LB_GPUTPCGMMergerHitWeights_resolveShared 256 + #endif #ifndef GPUCA_LB_GPUTPCGMMergerMergeLoopers_step0 #define GPUCA_LB_GPUTPCGMMergerMergeLoopers_step0 256 #endif diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 8054f9427051d..c19b32ecc305c 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -140,12 +140,13 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") +AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") -AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following, 8: mirroring)") +AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression (0 = no, 1 = strategy A, 2 = strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits (1=truncate charge/width LSB, 2=differences, 4=track-model)") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx b/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx index fd3c03a8b29ec..cde58817d3afd 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx @@ -187,8 +187,8 @@ void GPUChainTracking::PrintMemoryRelations() GPUInfo("MEMREL SectorTrackHits NCl %d NTrkH %d", processors()->tpcTrackers[i].NHitsTotal(), *processors()->tpcTrackers[i].NTrackHits()); } if (processors()->tpcMerger.Memory()) { - GPUInfo("MEMREL Tracks NCl %d NTrk %d", processors()->tpcMerger.NMaxClusters(), processors()->tpcMerger.NMergedTracks()); - GPUInfo("MEMREL TrackHitss NCl %d NTrkH %d", processors()->tpcMerger.NMaxClusters(), processors()->tpcMerger.NMergedTrackClusters()); + GPUInfo("MEMREL Tracks NCl %d NTrk %d", processors()->tpcMerger.NClusters(), processors()->tpcMerger.NMergedTracks()); + GPUInfo("MEMREL TrackHitss NCl %d NTrkH %d", processors()->tpcMerger.NClusters(), processors()->tpcMerger.NMergedTrackClusters()); } } @@ -217,7 +217,7 @@ void GPUChainTracking::PrintKernelDebugOutput() void GPUChainTracking::PrintOutputStat() { int32_t nTracks = 0, nAttachedClusters = 0, nAttachedClustersFitted = 0, nAdjacentClusters = 0; - uint32_t nCls = GetProcessingSettings().doublePipeline ? mIOPtrs.clustersNative->nClustersTotal : processors()->tpcMerger.NMaxClusters(); + uint32_t nCls = GetProcessingSettings().doublePipeline ? mIOPtrs.clustersNative->nClustersTotal : processors()->tpcMerger.NClusters(); if (GetProcessingSettings().createO2Output > 1) { nTracks = mIOPtrs.nOutputTracksTPCO2; nAttachedClusters = mIOPtrs.nMergedTrackHits; diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 96053356d69c8..f9a13bc012582 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -195,12 +195,8 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) Merger.CheckCollectedTracks(); } - uint32_t maxId = Merger.NMaxClusters(); - if (maxId > Merger.NMaxClusters()) { - throw std::runtime_error("mNMaxClusters too small"); - } - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), maxId * sizeof(*MergerShadow.SharedCount())); - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), maxId * sizeof(*MergerShadow.ClusterAttachment())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), Merger.NClusters() * sizeof(*MergerShadow.SharedCount())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), Merger.NClusters() * sizeof(*MergerShadow.ClusterAttachment())); runKernel(GetGridAuto(0, deviceType)); CondWaitEvent(waitForTransfer, &mEvents->single); runKernel(GetGridAuto(0, deviceType)); @@ -230,6 +226,16 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); if (param().rec.tpc.rebuildTrackInFit) { + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.HitWeights(), Merger.NClusters() * sizeof(*MergerShadow.HitWeights())); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + for (int32_t i = 0; i < param().rec.tpc.rebuildTrackInFitClusterCandidates; i++) { + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), i); + if (i + 1 < param().rec.tpc.rebuildTrackInFitClusterCandidates) { + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), i); + } + } + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); } runKernel(GetGridAuto(0)); @@ -268,7 +274,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracksdEdx(), MergerShadow.MergedTracksdEdx(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracksdEdx()), outputStream, 0); } GPUMemCpy(RecoStep::TPCMerging, Merger.Clusters(), MergerShadow.Clusters(), Merger.NMergedTrackClusters() * sizeof(*Merger.Clusters()), outputStream, 0); - GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NMaxClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); } if (GetProcessingSettings().outputSharedClusterMap) { TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResOutputState(), outputStream, nullptr, waitEvent); @@ -358,7 +364,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } if (GetProcessingSettings().debugLevel >= 2) { - GPUInfo("TPC Merger Finished (output clusters %d / input clusters %d)", Merger.NMergedTrackClusters(), Merger.NClusters()); + GPUInfo("TPC Merger Finished (output clusters %d / input clusters %d)", Merger.NMergedTrackClusters(), Merger.NSectorHits()); } return 0; } diff --git a/GPU/GPUTracking/Global/GPUErrorCodes.h b/GPU/GPUTracking/Global/GPUErrorCodes.h index a4921f478b107..2080548c7775c 100644 --- a/GPU/GPUTracking/Global/GPUErrorCodes.h +++ b/GPU/GPUTracking/Global/GPUErrorCodes.h @@ -33,16 +33,16 @@ GPUCA_ERROR_CODE(13, ERROR_SECTORDATA_HITINROW_OVERFLOW, SectorRow, Value, Max) GPUCA_ERROR_CODE(14, ERROR_SECTORDATA_BIN_OVERFLOW, SectorRow, Value, Max) GPUCA_ERROR_CODE(15, ERROR_SECTORDATA_Z_OVERFLOW, Sector, Value) GPUCA_ERROR_CODE(16, ERROR_MERGER_HIT_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(17, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(18, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(19, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(20, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(21, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(22, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(23, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(24, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER - +GPUCA_ERROR_CODE(17, ERROR_MERGER_REBUILD_HIT_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(18, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(19, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(20, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(21, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(22, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(23, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(24, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(25, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER GPUCA_ERROR_CODE(26, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded GPUCA_ERROR_CODE(28, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index d411d23115a32..ff9b6159468ff 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -398,12 +398,17 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mTrackIDs, GPUCA_NSECTORS * mNMaxSingleSectorTracks); // UnpackResetIds - RefitSectorTracks - UnpackSectorGlobal memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mTrackSort, mNMaxTracks); // PrepareForFit0 - SortTracksQPt - PrepareForFit1 - PrepareForFit1 / Finalize0 - Finalize2 - computePointerWithAlignment(mem, mSharedCount, mNMaxClusters); + computePointerWithAlignment(mem, mSharedCount, mNClusters); // PrepareForFit0 - SortTracksQPt - PrepareForFit1 - PrepareForFit1 / Finalize0 - Finalize2 + computePointerWithAlignment(mem, mTrackSort, mNMaxTracks); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway - computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUCA_ROW_COUNT * Param().rec.tpc.rebuildTrackInFitClusterCandidates); + computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway + if (mRec->GetParam().rec.tpc.rebuildTrackInFit) { + computePointerWithAlignment(mem, mSharedCount2, mNClusters); + computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUCA_ROW_COUNT * mRec->GetParam().rec.tpc.rebuildTrackInFitClusterCandidates); + computePointerWithAlignment(mem, mTrackRebuildHelper, mNMaxTracks); + computePointerWithAlignment(mem, mHitWeights, mNClusters); + } memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 @@ -437,14 +442,14 @@ void* GPUTPCGMMerger::SetPointersOutput(void* mem) } } computePointerWithAlignment(mem, mClusters, mNMaxMergedTrackClusters); - computePointerWithAlignment(mem, mClusterAttachment, mNMaxClusters); + computePointerWithAlignment(mem, mClusterAttachment, mNClusters); return mem; } void* GPUTPCGMMerger::SetPointersOutputState(void* mem) { if ((mRec->GetRecoSteps() & gpudatatypes::RecoStep::Refit) || mRec->GetProcessingSettings().outputSharedClusterMap) { - computePointerWithAlignment(mem, mClusterStateExt, mNMaxClusters); + computePointerWithAlignment(mem, mClusterStateExt, mNClusters); } else { mClusterStateExt = nullptr; } @@ -497,33 +502,33 @@ void GPUTPCGMMerger::RegisterMemoryAllocation() void GPUTPCGMMerger::SetMaxData(const GPUTrackingInOutPointers& io) { mNTotalSectorTracks = 0; - mNClusters = 0; + mNSectorHits = 0; mNMaxSingleSectorTracks = 0; for (int32_t iSector = 0; iSector < NSECTORS; iSector++) { uint32_t ntrk = *mRec->GetConstantMem().tpcTrackers[iSector].NTracks(); mNTotalSectorTracks += ntrk; - mNClusters += *mRec->GetConstantMem().tpcTrackers[iSector].NTrackHits(); + mNSectorHits += *mRec->GetConstantMem().tpcTrackers[iSector].NTrackHits(); if (mNMaxSingleSectorTracks < ntrk) { mNMaxSingleSectorTracks = ntrk; } } - mNMaxMergedTrackClusters = mRec->MemoryScalers()->NTPCMergedTrackHits(mNClusters); + mNMaxMergedTrackClusters = mRec->MemoryScalers()->NTPCMergedTrackHits(mNSectorHits); if (CAMath::Abs(Param().polynomialField.GetNominalBz()) < (gpu_common_constants::kZeroFieldCut * gpu_common_constants::kCLight)) { mNMaxTracks = mRec->MemoryScalers()->getValue(mNTotalSectorTracks, mNTotalSectorTracks); // 0 magnetic field } else { mNMaxTracks = mRec->MemoryScalers()->NTPCMergedTracks(mNTotalSectorTracks); } if (io.clustersNative) { - mNMaxClusters = io.clustersNative->nClustersTotal; + mNClusters = io.clustersNative->nClustersTotal; } else if (mRec->GetRecoSteps() & gpudatatypes::RecoStep::TPCSectorTracking) { - mNMaxClusters = 0; + mNClusters = 0; for (int32_t i = 0; i < NSECTORS; i++) { - mNMaxClusters += mRec->GetConstantMem().tpcTrackers[i].NHitsTotal(); + mNClusters += mRec->GetConstantMem().tpcTrackers[i].NHitsTotal(); } } else { - mNMaxClusters = mNClusters; + mNClusters = mNSectorHits; } - mNMaxLooperMatches = mNMaxClusters / 4; // We have that much scratch memory anyway + mNMaxLooperMatches = mNClusters / 4; // We have that much scratch memory anyway } int32_t GPUTPCGMMerger::CheckSectors() @@ -1667,7 +1672,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread const GPUTPCTracker& trk = GetConstantMem()->tpcTrackers[t->Sector()]; const GPUTPCHitId& ic = trk.TrackHits()[t->OrigTrack()->FirstHitID() + i]; uint32_t id = trk.Data().ClusterDataIndex(trk.Data().Row(ic.RowIndex()), ic.HitIndex()) + GetConstantMem()->ioPtrs.clustersNative->clusterOffset[t->Sector()][0]; - *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f}; + *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f, .weight = 0, .best = 0}; } nHits += nTrackHits; } @@ -1848,11 +1853,13 @@ GPUd() void GPUTPCGMMerger::PrepareForFit1(int32_t nBlocks, int32_t nThreads, in GPUTPCGMMergedTrack& trk = mMergedTracks[i]; if (trk.OK()) { for (uint32_t j = 0; j < trk.NClusters(); j++) { - uint32_t weight = attachAttached | attachGood; - if (CAMath::Abs(trk.GetParam().GetQPt() * Param().qptB5Scaler) <= Param().rec.tpc.rejectQPtB5 && !trk.MergedLooper() && trk.Leg() == 0) { - weight |= attachProtect; + if (!Param().rec.tpc.rebuildTrackInFit) { + uint32_t weight = attachAttached | attachGood; + if (CAMath::Abs(trk.GetParam().GetQPt() * Param().qptB5Scaler) <= Param().rec.tpc.rejectQPtB5 && !trk.MergedLooper() && trk.Leg() == 0) { + weight |= attachProtect; + } + mClusterAttachment[mClusters[trk.FirstClusterRef() + j].num] = weight; } - mClusterAttachment[mClusters[trk.FirstClusterRef() + j].num] = weight; CAMath::AtomicAdd(&mSharedCount[mClusters[trk.FirstClusterRef() + j].num], 1u); } if (!trk.CCE() && !trk.MergedLooper()) { @@ -1883,8 +1890,8 @@ GPUd() void GPUTPCGMMerger::PrepareForFit2(int32_t nBlocks, int32_t nThreads, in mClusters[i].state |= GPUTPCGMMergedTrackHit::flagShared; } } - if (mClusterStateExt) { - for (uint32_t i = iBlock * nThreads + iThread; i < mNMaxClusters; i += nBlocks * nThreads) { + if (!Param().rec.tpc.rebuildTrackInFit && mClusterStateExt) { + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { uint8_t state = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[i].getFlags(); if (mSharedCount[i] > 1) { state |= GPUTPCGMMergedTrackHit::flagShared; @@ -1935,7 +1942,7 @@ GPUd() void GPUTPCGMMerger::Finalize1(int32_t nBlocks, int32_t nThreads, int32_t GPUd() void GPUTPCGMMerger::Finalize2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) { - for (uint32_t i = iBlock * nThreads + iThread; i < mNMaxClusters; i += nThreads * nBlocks) { + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nThreads * nBlocks) { if (mClusterAttachment[i] != 0) { mClusterAttachment[i] = (mClusterAttachment[i] & attachFlagMask) | mTrackSort[mClusterAttachment[i] & attachTrackMask]; } @@ -2130,3 +2137,168 @@ GPUd() void GPUTPCGMMerger::MergeLoopersMain(int32_t nBlocks, int32_t nThreads, } #endif*/ } + +GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { + mSharedCount2[i] = 0; + } + if (iThread == 0 && iBlock == 0) { + mMemory->nMergedTrackClusters = 0; + } + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + const auto& trk = mMergedTracks[i]; + if (!trk.OK()) { + continue; + } + mTrackRebuildHelper[i].reverse = mClusters[trk.FirstClusterRef()].row < mClusters[trk.FirstClusterRef() + trk.NClusters() - 1].row; + int lastRow = -1; + for (uint32_t j = 0; j < trk.NClusters(); j++) { + const auto& cl = mClusters[trk.FirstClusterRef() + j]; + auto* candidates = &mClusterCandidates[(i * GPUCA_ROW_COUNT + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { + candidates[0].id = cl.num + 2; + candidates[0].best = 128; + candidates[0].weight = cl.state; + candidates[0].sector = cl.sector; + lastRow = cl.row; + } + } + } +} + +GPUd() void GPUTPCGMMerger::ComputeHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration) +{ + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + const auto& trk = mMergedTracks[i]; + if (!trk.OK() || trk.GetParam().GetNDF() <= 0) { + continue; + } + for (uint32_t j = 0; j < GPUCA_ROW_COUNT; j++) { + auto* candidates = &mClusterCandidates[(i * GPUCA_ROW_COUNT + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].best == 0) { + if (candidates[iteration].id >= 2) { + float weightf = trk.NClusters() * (trk.NClusters() * 2 - 5) * 128 / trk.GetParam().GetChi2() / (1.f + candidates[iteration].error); + int32_t weight = weightf < 0.f ? 0 : (weightf > 2e9f ? 2e9f : (int32_t)weightf); + candidates[iteration].weight = weight; + CAMath::AtomicMax(&mHitWeights[candidates[iteration].id - 2], weight); + CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, candidates[iteration].id - 2, weight)); + } else { + candidates[0].best = -1; + } + } + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration) +{ + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + auto& trk = mMergedTracks[i]; + if (!trk.OK() || trk.GetParam().GetNDF() <= 0) { + continue; + } + for (unsigned int j = 0; j < GPUCA_ROW_COUNT; j++) { + auto* candidates = &mClusterCandidates[(i * GPUCA_ROW_COUNT + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].best == 0) { + if (candidates[iteration].weight == mHitWeights[candidates[iteration].id - 2]) { + candidates[0].best = iteration + 1; + } + } + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + int32_t nIter = Param().rec.tpc.rebuildTrackInFitClusterCandidates; + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + auto& trk = mMergedTracks[i]; + if (!trk.OK()) { + trk.SetNClusters(0); + continue; + } + uint32_t nHits = 0; + uint8_t sharedRowCandidates[GPUCA_ROW_COUNT]; + uint8_t nSharedRowCandidates = 0; + for (unsigned int j = 0; j < GPUCA_ROW_COUNT; j++) { + auto* candidates = &mClusterCandidates[(i * GPUCA_ROW_COUNT + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (trk.GetParam().GetNDF() > 0 && candidates[0].best == 0 && candidates[nIter - 1].weight == mHitWeights[candidates[nIter - 1].id - 2]) { + candidates[0].best = nIter; + } + if (candidates[0].best > 0) { + nHits++; + } else if (candidates[0].id >= 2) { + sharedRowCandidates[nSharedRowCandidates++] = j; + } + } + + if (nSharedRowCandidates) { + const uint32_t nShared = CAMath::Min(nSharedRowCandidates, CAMath::Float2UIntRn(nHits * Param().rec.tpc.rebuildTrackMaxSharedFraction)); + if (nShared) { + const uint32_t n = Param().rec.tpc.rebuildTrackInFitClusterCandidates; + const auto* candidates = &mClusterCandidates[i * GPUCA_ROW_COUNT * n]; + GPUCommonAlgorithm::sort(sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [n, candidates](const uint8_t& a, const uint8_t& b) { + const auto& ca = candidates[a * n]; + const auto& cb = candidates[b * n]; + return GPUCA_DETERMINISTIC_CODE(ca.error == cb.error ? ca.id < cb.id : ca.error < cb.error, ca.error < cb.error); + }); + for (uint32_t j = 0; j < nShared; j++) { + auto* cj = &mClusterCandidates[(i * GPUCA_ROW_COUNT + sharedRowCandidates[j]) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + CADEBUG(printf("REBUILD: iTrk %d Recovering shared %d / %d: row %d %d, error %f\n", i, j, nShared, (int)sharedRowCandidates[j], cj[0].id - 2, cj[0].error)); + cj[0].best = 1; + mSharedCount2[cj[0].id - 2] = 1; // TODO: Should we also recover iter0 clusters? + } + nHits += nShared; + } + } + + const uint32_t iOut = CAMath::AtomicAdd(&mMemory->nMergedTrackClusters, nHits); + if (iOut + nHits > mNMaxMergedTrackClusters) { + raiseError(GPUErrors::ERROR_MERGER_REBUILD_HIT_OVERFLOW, iOut, mNMaxMergedTrackClusters); + CAMath::AtomicExch(&mMemory->nMergedTrackClusters, mNMaxMergedTrackClusters); + trk.SetNClusters(0); + trk.SetOK(false); + continue; + } + + trk.SetNClusters(nHits); + trk.SetFirstClusterRef(iOut); + uint32_t written = 0; + const bool reverse = mTrackRebuildHelper[i].reverse; + for (unsigned int j = 0; j < GPUCA_ROW_COUNT; j++) { + const auto& candidates = &mClusterCandidates[(i * GPUCA_ROW_COUNT + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].best > 0) { + auto& outCl = mClusters[iOut + (reverse ? (written) : (nHits - 1 - written))]; + if (candidates[0].best == 128) { + outCl = {.num = candidates[0].id - 2, .sector = candidates[0].sector, .row = (uint8_t)j, .state = (uint8_t)candidates[0].weight}; + written++; + continue; + } + const auto& best = candidates[candidates[0].best - 1]; + const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[best.id - 2]; + outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags)}; + written++; + CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, best.id - 2)); + } + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + for (uint32_t i = iBlock * nThreads + iThread; i < mMemory->nMergedTrackClusters; i += nBlocks * nThreads) { + if (mSharedCount2[mClusters[i].num]) { + mClusters[i].state |= GPUTPCGMMergedTrackHit::flagShared; // TODO: What is the best criterion to set the shared flag? Perhaps already when iter0 best hit would be shared? + } + } + if (mClusterStateExt) { + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { + uint8_t state = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[i].getFlags(); + if (mSharedCount2[i]) { + state |= GPUTPCGMMergedTrackHit::flagShared; + } + mClusterStateExt[i] = state; + } + } +} diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index f1e0dd0590c90..a9794b456e818 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -82,6 +82,12 @@ class GPUTPCGMMerger : public GPUProcessor uint8_t row; uint8_t sector; float error; + int32_t weight; + int32_t best; + }; + + struct trackRebuildHelper { + bool reverse; }; struct tmpSort { @@ -109,14 +115,15 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() GPUdEdxInfo* MergedTracksdEdx() { return mMergedTracksdEdx; } GPUhdi() const GPUdEdxInfo* MergedTracksdEdxAlt() const { return mMergedTracksdEdxAlt; } GPUhdi() GPUdEdxInfo* MergedTracksdEdxAlt() { return mMergedTracksdEdxAlt; } + GPUhdi() uint32_t NSectorHits() const { return mNSectorHits; } GPUhdi() uint32_t NClusters() const { return mNClusters; } - GPUhdi() uint32_t NMaxClusters() const { return mNMaxClusters; } GPUhdi() uint32_t NMaxTracks() const { return mNMaxTracks; } GPUhdi() uint32_t NMaxMergedTrackClusters() const { return mNMaxMergedTrackClusters; } GPUhdi() uint32_t NMergedTrackClusters() const { return mMemory->nMergedTrackClusters; } GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } - GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return (mClusters); } - GPUhdi() trackCluster* ClusterCandidates() { return (mClusterCandidates); } + GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return mClusters; } + GPUhdi() trackCluster* ClusterCandidates() { return mClusterCandidates; } + GPUhdi() int32_t* HitWeights() { return mHitWeights; } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } @@ -129,6 +136,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() tmpSort* TrackSortO2() { return mTrackSortO2; } GPUhdi() internal::MergeLooperParam* LooperCandidates() { return mLooperCandidates; } GPUhdi() GPUAtomic(uint32_t) * SharedCount() { return mSharedCount; } + GPUhdi() uint8_t* SharedCount2() { return mSharedCount2; } GPUhdi() gputpcgmmergertypes::GPUTPCGMBorderRange* BorderRange(int32_t i) { return mBorderRange[i]; } GPUhdi() const gputpcgmmergertypes::GPUTPCGMBorderRange* BorderRange(int32_t i) const { return mBorderRange[i]; } GPUhdi() GPUTPCGMBorderTrack* BorderTracks(int32_t i) { return mBorder[i]; } @@ -186,6 +194,11 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void MergeLoopersInit(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void MergeLoopersSort(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void MergeLoopersMain(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void ComputeHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); + GPUd() void ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); + GPUd() void ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); #ifndef GPUCA_GPUCODE void DumpSectorTracks(std::ostream& out) const; @@ -249,7 +262,7 @@ class GPUTPCGMMerger : public GPUProcessor uint32_t mNMaxTracks = 0; // maximum number of output tracks uint32_t mNMaxSingleSectorTracks = 0; // max N tracks in one sector uint32_t mNMaxMergedTrackClusters = 0; // max number of clusters in output tracks (double-counting shared clusters) - uint32_t mNMaxClusters = 0; // max total unique clusters (in event) + uint32_t mNClusters = 0; // max total unique clusters (in event) uint32_t mNMaxLooperMatches = 0; // Maximum number of candidate pairs for looper matching uint16_t mMemoryResMemory = (uint16_t)-1; @@ -260,9 +273,12 @@ class GPUTPCGMMerger : public GPUProcessor uint16_t mMemoryResOutputO2MC = (uint16_t)-1; uint16_t mMemoryResOutputO2Scratch = (uint16_t)-1; - int32_t mNClusters = 0; // Total number of incoming clusters (from sector tracks) + int32_t mNSectorHits = 0; // Total number of incoming clusters (from sector tracks) GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks trackCluster* mClusterCandidates = nullptr; + trackRebuildHelper* mTrackRebuildHelper = nullptr; + int32_t* mHitWeights = nullptr; + GPUdEdxInfo* mMergedTracksdEdx = nullptr; //* dEdx information GPUdEdxInfo* mMergedTracksdEdxAlt = nullptr; //* dEdx alternative information GPUTPCGMSectorTrack* mSectorTrackInfos = nullptr; //* additional information for sector tracks @@ -283,6 +299,7 @@ class GPUTPCGMMerger : public GPUProcessor uint32_t* mTrackSort = nullptr; tmpSort* mTrackSortO2 = nullptr; GPUAtomic(uint32_t) * mSharedCount = nullptr; // Must be uint32_t unfortunately for atomic support + uint8_t* mSharedCount2 = nullptr; GPUTPCGMBorderTrack* mBorderMemory = nullptr; // memory for border tracks GPUTPCGMBorderTrack* mBorder[2 * NSECTORS]; gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRangeMemory = nullptr; // memory for border tracks diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index 0a83bf47f5725..7e9c7529f450d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -173,9 +173,8 @@ void GPUTPCGMMerger::DumpFitPrepare(std::ostream& out) const } out << "\n"; } - uint32_t maxId = mNMaxClusters; uint32_t j = 0; - for (uint32_t i = 0; i < maxId; i++) { + for (uint32_t i = 0; i < mNClusters; i++) { if ((mClusterAttachment[i] & attachFlagMask) != 0) { if (++j % 10 == 0) { out << " Cluster attachment "; @@ -237,9 +236,8 @@ void GPUTPCGMMerger::DumpFinal(std::ostream& out) const } out << "\n"; } - uint32_t maxId = mNMaxClusters; uint32_t j = 0; - for (uint32_t i = 0; i < maxId; i++) { + for (uint32_t i = 0; i < mNClusters; i++) { if ((mClusterAttachment[i] & attachFlagMask) != 0) { if (++j % 10 == 0) { out << " Cluster attachment "; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 0b2229a33d0fc..7ac1c06992bf4 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -214,3 +214,34 @@ GPUdii() void GPUTPCGMMergerMergeLoopers::Thread<2>(int32_t nBlocks, int32_t nTh { merger.MergeLoopersMain(nBlocks, nThreads, iBlock, iThread); } + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.PrepareHitWeights(nBlocks, nThreads, iBlock, iThread); +} + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ComputeHitWeights(nBlocks, nThreads, iBlock, iThread, iteration); +} + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ResolveHitWeights1(nBlocks, nThreads, iBlock, iThread, iteration); +} + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ResolveHitWeights2(nBlocks, nThreads, iBlock, iThread); +} + + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ResolveHitWeightsShared(nBlocks, nThreads, iBlock, iThread); +} diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index 0e3e4990ff04b..669c241c236d2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -180,6 +180,19 @@ class GPUTPCGMMergerMergeLoopers : public GPUTPCGMMergerGeneral GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger); }; +class GPUTPCGMMergerHitWeights : public GPUTPCGMMergerGeneral +{ + public: + enum K { defaultKernel = 0, + prepare = 0, + compute = 1, + resolve1 = 2, + resolve2 = 3, + resolveShared = 4 }; + template + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t iteration); +}; + } // namespace o2::gpu #endif diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index e92610918c3f4..26b04f46130f5 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -137,26 +137,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ // CADEBUG(if ((uint32_t)merger->GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger->GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); // clang-format on - if (rebuilt && merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id > 0) { - while (true) { - if (!(ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector)) { - break; - } - } - const auto& interHit = merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; - if (interHit.id == 1) { - nMissed++; - nMissed2++; - continue; - } - const ClusterNative& GPUrestrict() cl = merger -> GetConstantMem()->ioPtrs.clustersNative->clustersLinear[interHit.id - 2]; - merger->GetConstantMem()->calibObjects.fastTransformHelper->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); - if (interHit.id - 2 == clusters[ihit].num) { - clusterState = clusters[ihit].state; - } else { - clusterState = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; - } - } else if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { + if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; nMissed2++; continue; @@ -228,7 +209,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - const float zOffset = merger->GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin); + const float zOffset = param.par.continuousTracking ? merger->GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = row.Grid().YMin(); const float stepY = row.HstepY(); const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway @@ -304,13 +285,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; } - merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err}; + merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err, .weight = 0, .best = 0}; nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); } } } } } + CADEBUG(const auto* dbgCand = &merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Hit %d Candidate %d hit %d err %f\n", iTrk, (int)cluster.row, cluster.num, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); } if (nCandidates == 0) { merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; @@ -1043,9 +1025,13 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger->MergedTracks()[iTrk].Looper()); }); - track.SetOK(ok); - track.Param() = t; - track.Alpha() = Alpha; + track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? + if (t.GetNDF() <= 0 && !rebuilt && merger->Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? + track.Param().NDF() = 0; + } else { + track.Param() = t; + track.Alpha() = Alpha; + } if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) { track.SetNClustersFitted(nTrackHits); } diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h index 66216181fdee6..891719ea74263 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h @@ -162,9 +162,12 @@ class GPUTPCTracker : public GPUProcessor GPUdi() static int32_t CalculateHitWeight(int32_t NHits, float chi2) { float weight = NHits * (NHits * 2 - 5) * 128 / chi2; // TODO: Add QPt to this formula - if (weight < 0.f || weight > 2e9f) { + if (weight < 0.f) { return 0; } + if (weight > 2e9f) { + return 2e9f; + } return ((int32_t)weight); } GPUdi() float GetChiSeedFactor() const { return Param().rec.tpc.hitPickUpFactor * Param().rec.tpc.hitPickUpFactor * 3.5f * 3.5f; } diff --git a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx index b9825bc6da481..78bc41ec7350a 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx +++ b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx @@ -914,7 +914,7 @@ int32_t main(int argc, char** argv) nEventsProcessed++; if (configStandalone.timeFrameTime) { - double nClusters = chainTracking->GetProcessors()->tpcMerger.NMaxClusters(); + double nClusters = chainTracking->GetProcessors()->tpcMerger.NClusters(); if (nClusters > 0) { const int32_t nOrbits = 32; const double colRate = 50000; diff --git a/GPU/GPUTracking/kernels.cmake b/GPU/GPUTracking/kernels.cmake index 8d8d2102c3b6c..a9d0de63a7c70 100644 --- a/GPU/GPUTracking/kernels.cmake +++ b/GPU/GPUTracking/kernels.cmake @@ -84,6 +84,11 @@ o2_gpu_add_kernel("GPUTPCGMMergerFinalize, step2" "GPUTP o2_gpu_add_kernel("GPUTPCGMMergerMergeLoopers, step0" "GPUTPCGMMergerGPU TPCMERGER" LB) o2_gpu_add_kernel("GPUTPCGMMergerMergeLoopers, step1" "GPUTPCGMMergerGPU TPCMERGER" LB) o2_gpu_add_kernel("GPUTPCGMMergerMergeLoopers, step2" "GPUTPCGMMergerGPU TPCMERGER" LB) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, prepare" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, compute" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, resolve1" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, resolve2" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, resolveShared" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) o2_gpu_add_kernel("GPUTPCGMO2Output, prepare" "= TPCMERGER" LB) o2_gpu_add_kernel("GPUTPCGMO2Output, sort" "= TPCMERGER") o2_gpu_add_kernel("GPUTPCGMO2Output, output" "= TPCMERGER" LB) diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 852ac5c1feefb..e52040e714246 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -1822,7 +1822,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } } - uint32_t nCl = clNative ? clNative->nClustersTotal : mTracking->GetProcessors()->tpcMerger.NMaxClusters(); + uint32_t nCl = clNative ? clNative->nClustersTotal : mTracking->GetProcessors()->tpcMerger.NClusters(); mClusterCounts.nTotal += nCl; if (mQATasks & (taskClusterCounts | taskClusterRejection)) { for (uint32_t iSector = 0; iSector < GPUCA_NSECTORS; iSector++) { From ff97b9dc004436abed0416cc225c4fa9b084e5bd Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 15 Oct 2025 09:44:10 +0200 Subject: [PATCH 06/25] GPU: Additional debug dumps --- .../Global/GPUChainTrackingMerger.cxx | 2 + GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 4 + GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 87 ++++++++++++++----- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 1 - 4 files changed, 71 insertions(+), 23 deletions(-) diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index f9a13bc012582..bf6d659adf86f 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -235,7 +235,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpInterpolatedHits, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRebuiltTracks, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); } runKernel(GetGridAuto(0)); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index a9794b456e818..0d2b190203450 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -212,6 +212,10 @@ class GPUTPCGMMerger : public GPUProcessor void DumpRefit(std::ostream& out) const; void DumpFinal(std::ostream& out) const; void DumpLoopers(std::ostream& out) const; + void DumpInterpolatedHits(std::ostream& out) const; + void DumpRebuiltTracks(std::ostream& out) const; + void DumpTrackParam(std::ostream& out) const; + void DumpTrackClusters(std::ostream& out, bool non0StateOnly = false, bool noNDF0 = false) const; template void MergedTrackStreamerInternal(const GPUTPCGMBorderTrack& b1, const GPUTPCGMBorderTrack& b2, const char* name, int32_t sector1, int32_t sector2, int32_t mergeMode, float weight, float frac) const; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index 7e9c7529f450d..4d31147227a95 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -133,10 +133,15 @@ void GPUTPCGMMerger::DumpMergedBetweenSectors(std::ostream& out) const } void GPUTPCGMMerger::DumpCollected(std::ostream& out) const +{ + out << "\nTPC Merger Collected Tracks\n"; + DumpTrackParam(out); +} + +void GPUTPCGMMerger::DumpTrackParam(std::ostream& out) const { std::streamsize ss = out.precision(); out << std::setprecision(6); - out << "\nTPC Merger Collected Tracks\n"; for (uint32_t i = 0; i < mMemory->nMergedTracks; i++) { const auto& trk = mMergedTracks[i]; const auto& p = trk.GetParam(); @@ -145,6 +150,15 @@ void GPUTPCGMMerger::DumpCollected(std::ostream& out) const out << std::setprecision(ss); } +void GPUTPCGMMerger::DumpRebuiltTracks(std::ostream& out) const +{ + out << "\nTPC Merger Rebuilt Tracks\n"; + out << " Cluster Attachment\n"; + DumpTrackClusters(out, false, true); + out << " Track Params\n"; + DumpTrackParam(out); +} + void GPUTPCGMMerger::DumpMergeCE(std::ostream& out) const { DumpTrackLinks(out, true, " for CE merging"); @@ -157,6 +171,26 @@ void GPUTPCGMMerger::DumpMergeCE(std::ostream& out) const } } +void GPUTPCGMMerger::DumpTrackClusters(std::ostream& out, bool non0StateOnly, bool noNDF0) const +{ + for (uint32_t j = 0; j < mMemory->nMergedTracks; j++) { + const auto& trk = mMergedTracks[j]; + if (trk.NClusters() == 0) { + continue; + } + if (noNDF0 && (!trk.OK() || trk.GetParam().GetNDF() < 0)) { + continue; + } + out << " Track " << j << ": (" << trk.NClusters() << "): "; + for (uint32_t i = trk.FirstClusterRef(); i < trk.FirstClusterRef() + trk.NClusters(); i++) { + if (!non0StateOnly || mClusters[i].state != 0) { + out << j << "/" << (i - trk.FirstClusterRef()) << ": " << (int32_t)mClusters[i].row << "/" << mClusters[i].num << "/" << (int32_t)mClusters[i].state << ", "; + } + } + out << "\n"; + } +} + void GPUTPCGMMerger::DumpFitPrepare(std::ostream& out) const { out << "\nTPC Merger Refit Prepare\n"; @@ -165,14 +199,7 @@ void GPUTPCGMMerger::DumpFitPrepare(std::ostream& out) const out << " " << i << ": " << mTrackOrderAttach[i] << "\n"; } out << " Clusters\n"; - for (uint32_t j = 0; j < mMemory->nMergedTracks; j++) { - const auto& trk = mMergedTracks[j]; - out << " Track " << j << ": "; - for (uint32_t i = trk.FirstClusterRef(); i < trk.FirstClusterRef() + trk.NClusters(); i++) { - out << j << "/" << (i - trk.FirstClusterRef()) << ": " << mClusters[i].num << "/" << (int32_t)mClusters[i].state << ", "; - } - out << "\n"; - } + DumpTrackClusters(out); uint32_t j = 0; for (uint32_t i = 0; i < mNClusters; i++) { if ((mClusterAttachment[i] & attachFlagMask) != 0) { @@ -223,19 +250,7 @@ void GPUTPCGMMerger::DumpLoopers(std::ostream& out) const void GPUTPCGMMerger::DumpFinal(std::ostream& out) const { out << "\nTPC Merger Finalized\n"; - for (uint32_t j = 0; j < mMemory->nMergedTracks; j++) { - const auto& trk = mMergedTracks[j]; - if (trk.NClusters() == 0) { - continue; - } - out << " Track " << j << ": "; - for (uint32_t i = trk.FirstClusterRef(); i < trk.FirstClusterRef() + trk.NClusters(); i++) { - if (mClusters[i].state != 0) { - out << j << "/" << (i - trk.FirstClusterRef()) << ": " << mClusters[i].num << "/" << (int32_t)mClusters[i].state << ", "; - } - } - out << "\n"; - } + DumpTrackClusters(out, true); uint32_t j = 0; for (uint32_t i = 0; i < mNClusters; i++) { if ((mClusterAttachment[i] & attachFlagMask) != 0) { @@ -251,6 +266,34 @@ void GPUTPCGMMerger::DumpFinal(std::ostream& out) const out << "\n"; } +void GPUTPCGMMerger::DumpInterpolatedHits(std::ostream& out) const +{ + out << "\nTPC Merger Interpolated Hits\n"; + for (uint32_t i = 0; i < mMemory->nMergedTracks; i++) { + const auto& trk = mMergedTracks[i]; + if (trk.OK() && trk.GetParam().GetNDF() >= 0 && trk.NClusters()) { + out << "Track " << i << ":"; + for (uint32_t j = 0; j < GPUCA_ROW_COUNT; j++) { + auto* candidates = &mClusterCandidates[(i * GPUCA_ROW_COUNT + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].id) { + out << " Row " << j << ": "; + if (candidates[0].best) { + out << " Best " << candidates[0].best << " - "; + } + for (uint32_t k = 0; k < Param().rec.tpc.rebuildTrackInFitClusterCandidates; k++) { + if (candidates[k].id) { + out << k << ": id " << candidates[k].id << " err " << candidates[k].error << " weight " << candidates[k].weight << " - "; + } + } + out << "; "; + } + } + out << "\n"; + } + } + out << "\n"; +} + template inline void GPUTPCGMMerger::MergedTrackStreamerInternal(const GPUTPCGMBorderTrack& b1, const GPUTPCGMBorderTrack& b2, const char* name, int32_t sector1, int32_t sector2, int32_t mergeMode, float weight, float frac) const { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 7ac1c06992bf4..98f9c4081d519 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -239,7 +239,6 @@ GPUdii() void GPUTPCGMMergerHitWeights::Thread GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) { From 108982e90d357eb491f56872abc594996adc8583 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 6 Oct 2025 22:09:55 +0200 Subject: [PATCH 07/25] GPU TPC: Split TPC Merger Fit() into multiple functions --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 4 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 4 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 652 +++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 46 +- 4 files changed, 377 insertions(+), 329 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index ff9b6159468ff..aecd98c6a87be 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -571,7 +571,7 @@ GPUd() int32_t GPUTPCGMMerger::RefitSectorTrack(GPUTPCGMSectorTrack& sectorTrack trk.QPt() = inTrack->Param().GetQPt(); trk.TOffset() = Param().par.continuousTracking ? GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convZOffsetToVertexTime(sector, inTrack->Param().GetZOffset(), Param().continuousMaxTimeBin) : 0; const auto tmp = sectorTrack.ClusterTN() > sectorTrack.ClusterT0() ? std::array{sectorTrack.ClusterTN(), sectorTrack.ClusterT0()} : std::array{sectorTrack.ClusterT0(), sectorTrack.ClusterTN()}; - trk.ShiftZ(this, sector, tmp[0], tmp[1], inTrack->Param().GetX()); // We do not store the inner / outer cluster X, so we just use the track X instead + trk.ShiftZ(*this, sector, tmp[0], tmp[1], inTrack->Param().GetX()); // We do not store the inner / outer cluster X, so we just use the track X instead sectorTrack.SetX2(0.f); for (int32_t way = 0; way < 2; way++) { if (way) { @@ -1868,7 +1868,7 @@ GPUd() void GPUTPCGMMerger::PrepareForFit1(int32_t nBlocks, int32_t nThreads, in const auto& GPUrestrict() cls = GetConstantMem()->ioPtrs.clustersNative->clustersLinear; float z0 = cls[cl0.num].getTime(), zn = cls[cln.num].getTime(); const auto tmp = zn > z0 ? std::array{zn, z0, GPUTPCGeometry::Row2X(cln.row)} : std::array{z0, zn, GPUTPCGeometry::Row2X(cl0.row)}; - trk.Param().ShiftZ(this, cl0.sector, tmp[0], tmp[1], tmp[2]); + trk.Param().ShiftZ(*this, cl0.sector, tmp[0], tmp[1], tmp[2]); updTrk = &trk; while (updTrk->PrevSegment() >= 0) { auto next = &mMergedTracks[updTrk->PrevSegment()]; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 98f9c4081d519..12b50f9f4cee2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -23,7 +23,7 @@ GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThread { GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, rebuilt); + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt); }); } @@ -31,7 +31,7 @@ template <> GPUdii() void GPUTPCGMMergerFollowLoopers::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger) { GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), uint32_t, i, merger.Memory()->nLoopData, { - GPUTPCGMTrackParam::PropagateLooper(&merger, i); + GPUTPCGMTrackParam::PropagateLooper(merger, i); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 26b04f46130f5..0d4b76d8a7492 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -48,13 +48,13 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { - static constexpr float kDeg2Rad = M_PI / 180.f; static constexpr float maxSinPhi = GPUCA_MAX_SIN_PHI; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); - const GPUParam& GPUrestrict() param = merger->Param(); + const GPUParam& GPUrestrict() param = merger.Param(); + GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); GPUdEdx dEdx, dEdxAlt; GPUTPCGMPropagator prop; @@ -87,13 +87,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const bool refit = (nWays == 1 || iWay >= 1); const bool finalOutInFit = iWay + 2 >= nWays; const bool finalFit = iWay == nWays - 1; - const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); ResetCovariance(); prop.SetSeedingErrors(!(refit)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); - prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger->GetConstantMem()->calibObjects.matLUT : nullptr); + prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : GPUCA_MAX_SIN_PHI_LOW); CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / kSectAngle) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); @@ -129,13 +128,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; { - const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; - merger->GetConstantMem()->calibObjects.fastTransformHelper->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + merger.GetConstantMem()->calibObjects.fastTransformHelper->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); } // clang-format off CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - // CADEBUG(if ((uint32_t)merger->GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) - // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger->GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); + // CADEBUG(if ((uint32_t)merger.GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) + // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger.GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); // clang-format on if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; @@ -170,253 +169,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } } - int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - // clang-format off - CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); - // clang-format on - if (retValProp == -2) // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha - { - CADEBUG(printf("REROTATE\n")); - if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) == 0) { - retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - } - } - if (retValProp == 0) { - lastRow = cluster.row; - lastSector = cluster.sector; - } - // clang-format off - CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); - // clang-format on - - if (crossCE) { - if (param.rec.tpc.addErrorsCECrossing) { - if (param.rec.tpc.addErrorsCECrossing >= 2) { - AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); - } else { - AddCovDiagErrors(param.rec.tpc.errorsCECrossing); - } - } else if (mC[2] < 0.5f) { - mC[2] = 0.5f; - } - } - auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; - float uncorrectedY = -1e6f; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { - const GPUTPCTracker& GPUrestrict() tracker = *(merger -> GetConstantMem()->tpcTrackers + cluster.sector); - const GPUTPCRow& GPUrestrict() row = tracker.Row(cluster.row); - GPUglobalref() const cahit2* hits = tracker.HitData(row); - GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); - if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - const float zOffset = param.par.continuousTracking ? merger->GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin) : 0; - const float y0 = row.Grid().YMin(); - const float stepY = row.HstepY(); - const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway - const float stepZ = row.HstepZ(); - int32_t bin, ny, nz; - - float err2Y, err2Z; - param.GetClusterErrors2(cluster.sector, cluster.row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge - - const float Iz0 = inter.posY - mP[0]; - const float Iz1 = inter.posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; - - float uncorrectedZ; - merger->GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, ImP0, ImP1, uncorrectedY, uncorrectedZ); - - int32_t nCandidates = 0; - if (CAMath::Abs(uncorrectedY) <= row.getTPCMaxY()) { - const float kFactor = tracker.GetChiSeedFactor(); - const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? - const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); - const float tubeY = CAMath::Sqrt(sy2); - const float tubeZ = CAMath::Sqrt(sz2); - row.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); - - const int32_t nBinsY = row.Grid().Ny(); - const int32_t idOffset = tracker.Data().ClusterIdOffset(); - const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); - for (int32_t k = 0; k <= nz; k++) { - const int32_t mybin = bin + k * nBinsY; - const uint32_t hitFst = firsthit[mybin]; - const uint32_t hitLst = firsthit[mybin + ny + 1]; - for (uint32_t ih = hitFst; ih < hitLst; ih++) { - const cahit2 hh = hits[ih]; - const float y = y0 + hh.x * stepY; - const float z = z0 + hh.y * stepZ; - const float dy = y - uncorrectedY; - const float dz = z - uncorrectedZ; - - if (dy * dy < sy2 && dz * dz < sz2) { - float err2YA, err2ZA; - const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[idOffset + ids[ih]]; - const auto clflags = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; - const float time = cl.getTime(); - const float invSqrtCharge = CAMath::InvSqrt(cl.qMax); - const float invCharge = 1.f / cl.qMax; - float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); - invAvgCharge *= invAvgCharge; - - prop.GetErr2(err2YA, err2ZA, merger->Param(), mP[1], cluster.row, clflags, cluster.sector, time, invAvgCharge, invCharge); - const float Jw0 = 1.f / (ImC0 + err2YA); - const float Jw2 = 1.f / (ImC2 + err2ZA); - const float chi2Y = Jw0 * dy * dy; - const float chi2Z = Jw2 * dz * dz; - bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); - float err = dy * dy + dz * dz; - if (ok) { - int32_t insert = nCandidates; - for (int32_t c = 0; c < nCandidates; c++) { - if (err < merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { - insert = c; - break; - } - } - if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { - for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { - merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; - } - merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err, .weight = 0, .best = 0}; - nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); - } - } - } - } - } - CADEBUG(const auto* dbgCand = &merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Hit %d Candidate %d hit %d err %f\n", iTrk, (int)cluster.row, cluster.num, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); - } - if (nCandidates == 0) { - merger->ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; - } - } - } else if (allowChangeClusters) { - uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); - } - const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; - if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + int32_t retValHit = FitHit(merger, iTrk, track, N, NTolerated, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, lastRow, lastSector, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, ihitStart, allowChangeClusters, refit, finalFit, nMissed, nMissed2, covYYUpd, resetT0); + if (retValHit == 1) { break; - } - if (retValProp || sinPhiErr) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; - NTolerated++; - CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); + } else if (retValHit == -1) { continue; } - CADEBUG(printf("\n")); - - int32_t retValUpd = 0, retValInt = 0; - float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); - if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { - retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; - } else { - int8_t rejectChi2 = 0; - if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { - if (iWay == nWays - 3) { - rejectChi2 = GPUTPCGMPropagator::rejectInterFill; - } else if (iWay == nWays - 2) { - rejectChi2 = GPUTPCGMPropagator::rejectInterReject; - } else if (iWay == nWays - 1) { - rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; - } - } - - float err2Y, err2Z; - const float time = merger->GetConstantMem()->ioPtrs.clustersNative ? merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; - const float invSqrtCharge = merger->GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; - const float invCharge = merger->GetConstantMem()->ioPtrs.clustersNative ? (1.f / merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; - float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; - invAvgCharge *= invAvgCharge; - prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - - if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { - if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = GPUTPCGMPropagator::rejectDirect; - } else { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); - } - } - - if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters - retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedEdge; - } else { - retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); - } - GPUCA_DEBUG_STREAMER_CHECK(if (o2::utils::DebugStreamer::checkStream(o2::utils::StreamFlags::streamUpdateTrack, iTrk)) { - merger->DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, inter, rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); - }); - } - // clang-format off - CADEBUG(if (!CheckCov()) GPUError("INVALID COV AFTER UPDATE!!!")); - CADEBUG(printf("\t%21sFit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd, retValInt)); - // clang-format on - - ConstrainSinPhi(); // TODO: Limit using ConstrainSinPhi everywhere! - if (!retValUpd && !retValInt) // track is updated - { - lastUpdateX = mX; - covYYUpd = mC[0]; - nMissed = nMissed2 = 0; - UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - N++; - ihitStart = ihit; - float dy = mP[0] - prop.Model().Y(); - float dz = mP[1] - prop.Model().Z(); - if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { - CADEBUG(printf("Reinit linearization\n")); - prop.SetTrack(this, prop.GetAlpha()); - } - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { - if (param.dodEdxEnabled && finalFit) { // TODO: Costimize flag to remove, and option to remove double-clusters - bool acc = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMask) == 0, accAlt = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMaskAlt) == 0; - if (acc || accAlt) { - float qtot = 0, qmax = 0, pad = 0, relTime = 0; - const int32_t clusterCount = (ihit - ihitMergeFirst) * wayDirection + 1; - for (int32_t iTmp = ihitMergeFirst; iTmp != ihit + wayDirection; iTmp += wayDirection) { - const ClusterNative& cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num]; - qtot += cl.qTot; - qmax = CAMath::Max(qmax, cl.qMax); - pad += cl.getPad(); - relTime += cl.getTime(); - } - qtot /= clusterCount; // TODO: Weighted Average - pad /= clusterCount; - relTime /= clusterCount; - relTime = relTime - CAMath::Round(relTime); - if (acc) { - dEdx.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], merger->GetConstantMem()->calibObjects, zz, pad, relTime); - } - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - if (accAlt) { - dEdxAlt.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], merger->GetConstantMem()->calibObjects, zz, pad, relTime); - } - } - } - } - } - } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track - if (retValInt || allowChangeClusters) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); - } else if (finalFit) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); - } - if (!retValInt) { - nMissed++; - nMissed2++; - } - } else { - break; // bad chi2 for the whole track, stop the fit - } } if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); @@ -450,9 +210,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } if (param.par.dodEdx && param.dodEdxEnabled) { - dEdx.computedEdx(merger->MergedTracksdEdx()[iTrk], param); + dEdx.computedEdx(merger.MergedTracksdEdx()[iTrk], param); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - dEdxAlt.computedEdx(merger->MergedTracksdEdxAlt()[iTrk], param); + dEdxAlt.computedEdx(merger.MergedTracksdEdxAlt()[iTrk], param); } } Alpha = prop.GetAlpha(); @@ -462,6 +222,277 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ return true; } +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0) +{ + const GPUParam& GPUrestrict() param = merger.Param(); + const int32_t nWays = param.rec.tpc.nWays; + const int32_t wayDirection = (iWay & 1) ? -1 : 1; + const auto& cluster = clusters[ihit]; + int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); + // clang-format off + CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); + // clang-format on + if (retValProp == -2) // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha + { + CADEBUG(printf("REROTATE\n")); + if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) == 0) { + retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); + } + } + if (retValProp == 0) { + lastRow = cluster.row; + lastSector = cluster.sector; + } + // clang-format off + CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); + // clang-format on + + if (crossCE) { + if (param.rec.tpc.addErrorsCECrossing) { + if (param.rec.tpc.addErrorsCECrossing >= 2) { + AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); + } else { + AddCovDiagErrors(param.rec.tpc.errorsCECrossing); + } + } else if (mC[2] < 0.5f) { + mC[2] = 0.5f; + } + } + + float uncorrectedY = -1e6f; + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } else if (allowChangeClusters) { + uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); + } + + static constexpr float kDeg2Rad = M_PI / 180.f; + const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); + const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; + if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + return 1; + } + if (retValProp || sinPhiErr) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + nMissed2++; + NTolerated++; + CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); + return -1; + } + CADEBUG(printf("\n")); + + int32_t retValUpd = 0, retValInt = 0; + float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); + if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { + retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; + } else { + int8_t rejectChi2 = 0; + if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { + if (iWay == nWays - 3) { + rejectChi2 = GPUTPCGMPropagator::rejectInterFill; + } else if (iWay == nWays - 2) { + rejectChi2 = GPUTPCGMPropagator::rejectInterReject; + } else if (iWay == nWays - 1) { + rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; + } + } + + float err2Y, err2Z; + const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; + const float invSqrtCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; + const float invCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? (1.f / merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; + float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; + invAvgCharge *= invAvgCharge; + + prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); + + if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { + if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + rejectChi2 = GPUTPCGMPropagator::rejectDirect; + } else { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); + } + } + + if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters + retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedEdge; + } else { + retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); + } + GPUCA_DEBUG_STREAMER_CHECK(if (o2::utils::DebugStreamer::checkStream(o2::utils::StreamFlags::streamUpdateTrack, iTrk)) { + merger.DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, inter, rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); + }); + } + // clang-format off + CADEBUG(if (!CheckCov()) GPUError("INVALID COV AFTER UPDATE!!!")); + CADEBUG(printf("\t%21sFit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd, retValInt)); + // clang-format on + + ConstrainSinPhi(); // TODO: Limit using ConstrainSinPhi everywhere! + if (!retValUpd && !retValInt) // track is updated + { + lastUpdateX = mX; + covYYUpd = mC[0]; + nMissed = nMissed2 = 0; + UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + N++; + ihitStart = ihit; + float dy = mP[0] - prop.Model().Y(); + float dz = mP[1] - prop.Model().Z(); + if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { + CADEBUG(printf("Reinit linearization\n")); + prop.SetTrack(this, prop.GetAlpha()); + } + FitdEdx(dEdx, dEdxAlt, param, finalFit, merger.GetConstantMem()->calibObjects, ihit, ihitMergeFirst, wayDirection, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear, clusters, clusterState, zz); + } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track + if (retValInt || allowChangeClusters) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); + } else if (finalFit) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + } + if (!retValInt) { + nMissed++; + nMissed2++; + } + } else { + return 1; // bad chi2 for the whole track, stop the fit + } + return 0; +} + +GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +{ + const GPUParam& GPUrestrict() param = merger.Param(); + const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); + const GPUTPCRow& GPUrestrict() rowData = tracker.Row(row); + GPUglobalref() const cahit2* hits = tracker.HitData(rowData); + GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); + float uncorrectedY = -1e6f, uncorrectedZ; + if (rowData.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; + const float y0 = rowData.Grid().YMin(); + const float stepY = rowData.HstepY(); + const float z0 = rowData.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway + const float stepZ = rowData.HstepZ(); + int32_t bin, ny, nz; + + float err2Y, err2Z; + param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + + const float Iz0 = inter.posY - mP[0]; + const float Iz1 = inter.posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0; + const float ImP1 = mP[1] + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0]; + const float ImC2 = mC[2] - Ik11 * mC[2]; + + merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); + + int32_t nCandidates = 0; + if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { + const float kFactor = tracker.GetChiSeedFactor(); + const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? + const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); + const float tubeY = CAMath::Sqrt(sy2); + const float tubeZ = CAMath::Sqrt(sz2); + rowData.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); + + const int32_t nBinsY = rowData.Grid().Ny(); + const int32_t idOffset = tracker.Data().ClusterIdOffset(); + const int32_t* ids = &(tracker.Data().ClusterDataIndex()[rowData.HitNumberOffset()]); + for (int32_t k = 0; k <= nz; k++) { + const int32_t mybin = bin + k * nBinsY; + const uint32_t hitFst = firsthit[mybin]; + const uint32_t hitLst = firsthit[mybin + ny + 1]; + for (uint32_t ih = hitFst; ih < hitLst; ih++) { + const cahit2 hh = hits[ih]; + const float y = y0 + hh.x * stepY; + const float z = z0 + hh.y * stepZ; + const float dy = y - uncorrectedY; + const float dz = z - uncorrectedZ; + + if (dy * dy < sy2 && dz * dz < sz2) { + float err2YA, err2ZA; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[idOffset + ids[ih]]; + const auto clflags = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + const float time = cl.getTime(); + const float invSqrtCharge = CAMath::InvSqrt(cl.qMax); + const float invCharge = 1.f / cl.qMax; + float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); + invAvgCharge *= invAvgCharge; + + prop.GetErr2(err2YA, err2ZA, param, mP[1], row, clflags, sector, time, invAvgCharge, invCharge); + const float Jw0 = 1.f / (ImC0 + err2YA); + const float Jw2 = 1.f / (ImC2 + err2ZA); + const float chi2Y = Jw0 * dy * dy; + const float chi2Z = Jw2 * dz * dz; + bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); + float err = dy * dy + dz * dz; + if (ok) { + int32_t insert = nCandidates; + for (int32_t c = 0; c < nCandidates; c++) { + if (err < merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { + insert = c; + break; + } + } + if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { + for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { + merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; + } + merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = row, .sector = sector, .error = err, .weight = 0, .best = 0}; + nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); + } + } + } + } + } + CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); + } + if (nCandidates == 0) { + merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; + } + } + return uncorrectedY; +} + +GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, const GPUParam& GPUrestrict() param, bool finalFit, const GPUCalibObjectsConst& GPUrestrict() calib, int ihit, int ihitMergeFirst, int wayDirection, const ClusterNative* GPUrestrict() clustersArray, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz) +{ + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + if (param.dodEdxEnabled && finalFit) { // TODO: Costimize flag to remove, and option to remove double-clusters + bool acc = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMask) == 0, accAlt = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMaskAlt) == 0; + if (acc || accAlt) { + float qtot = 0, qmax = 0, pad = 0, relTime = 0; + const int32_t clusterCount = CAMath::Abs(ihit - ihitMergeFirst) + 1; + for (int32_t iTmp = ihitMergeFirst; iTmp != ihit + wayDirection; iTmp += wayDirection) { + const ClusterNative& cl = clustersArray[clusters[iTmp].num]; + qtot += cl.qTot; + qmax = CAMath::Max(qmax, cl.qMax); + pad += cl.getPad(); + relTime += cl.getTime(); + } + qtot /= clusterCount; // TODO: Weighted Average + pad /= clusterCount; + relTime /= clusterCount; + relTime = relTime - CAMath::Round(relTime); + const auto& cluster = clusters[ihit]; + if (acc) { + dEdx.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime); + } + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { + if (accAlt) { + dEdxAlt.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime); + } + } + } + } + } +} + GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& Alpha) { static constexpr float kDeg2Rad = M_PI / 180.f; @@ -519,11 +550,11 @@ GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, mChi2 = 0; } -GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger* GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) +GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) { if (ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector) { float maxDistY, maxDistZ; - prop.GetErr2(maxDistY, maxDistZ, merger->Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge + prop.GetErr2(maxDistY, maxDistZ, merger.Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge maxDistY = (maxDistY + mC[0]) * 20.f; maxDistZ = (maxDistZ + mC[2]) * 20.f; int32_t noReject = 0; // Cannot reject if simple estimation of y/z fails (extremely unlike case) @@ -538,10 +569,10 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t xx = yy = zz = 0.f; clusterState = 0; while (true) { - const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; float clamp = cl.qTot; float clx, cly, clz; - merger->GetConstantMem()->calibObjects.fastTransformHelper->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); + merger.GetConstantMem()->calibObjects.fastTransformHelper->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); float dy = cly - projY; float dz = clz - projZ; if (noReject == 0 && (dy * dy > maxDistY || dz * dz > maxDistZ)) { @@ -573,25 +604,25 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t return 0; } -GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop) +GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop) { float Y, Z; float X = 0; - Merger->GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoX(sector, iRow, mP[0], mP[1], X); + merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoX(sector, iRow, mP[0], mP[1], X); if (prop.GetPropagatedYZ(X, Y, Z)) { Y = mP[0]; Z = mP[1]; } - return AttachClusters(Merger, sector, iRow, iTrack, goodLeg, Y, Z); + return AttachClusters(merger, sector, iRow, iTrack, goodLeg, Y, Z); } -GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) +GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) { - const auto& param = Merger->Param(); + const GPUParam& GPUrestrict() param = merger.Param(); if (param.rec.tpc.disableRefitAttachment & 1) { return -1e6f; } - const GPUTPCTracker& GPUrestrict() tracker = *(Merger->GetConstantMem()->tpcTrackers + sector); + const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); const GPUTPCRow& GPUrestrict() row = tracker.Row(iRow); GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); @@ -599,7 +630,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric return -1e6f; } - const float zOffset = param.par.continuousTracking ? Merger->GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; // TODO: do some validatiomns for the transform conv functions... + const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; // TODO: do some validatiomns for the transform conv functions... const float y0 = row.Grid().YMin(); const float stepY = row.HstepY(); const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway @@ -607,7 +638,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric int32_t bin, ny, nz; float uncorrectedY, uncorrectedZ; - Merger->GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(sector, iRow, Y, Z, uncorrectedY, uncorrectedZ); + merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(sector, iRow, Y, Z, uncorrectedY, uncorrectedZ); if (CAMath::Abs(uncorrectedY) > row.getTPCMaxY()) { return uncorrectedY; } @@ -619,7 +650,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric const float tubeMinSize2 = protect ? param.rec.tpc.tubeProtectMinSize2 : 0.f; float tubeSigma2 = protect ? param.rec.tpc.tubeProtectSigma2 : param.rec.tpc.tubeRemoveSigma2; uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, uncorrectedY)); - float time = Merger->GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->InverseTransformInTimeFrame(sector, uncorrectedZ + (param.par.continuousTracking ? Merger->GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0), param.continuousMaxTimeBin); // TODO: Simplify this call in TPCFastTransform + float time = merger.GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->InverseTransformInTimeFrame(sector, uncorrectedZ + (param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0), param.continuousMaxTimeBin); // TODO: Simplify this call in TPCFastTransform if (iRow < param.rec.tpc.tubeExtraProtectMinRow || pad < param.rec.tpc.tubeExtraProtectEdgePads || pad >= (uint32_t)(GPUTPCGeometry::NPads(iRow) - param.rec.tpc.tubeExtraProtectEdgePads) || param.GetUnscaledMult(time) / GPUTPCGeometry::Row2X(iRow) > param.rec.tpc.tubeExtraProtectMinOccupancy) { @@ -636,8 +667,8 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric const int32_t nBinsY = row.Grid().Ny(); const int32_t idOffset = tracker.Data().ClusterIdOffset(); const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); - uint32_t myWeight = Merger->TrackOrderAttach()[iTrack] | gputpcgmmergertypes::attachAttached | gputpcgmmergertypes::attachTube; - GPUAtomic(uint32_t)* const weights = Merger->ClusterAttachment(); + uint32_t myWeight = merger.TrackOrderAttach()[iTrack] | gputpcgmmergertypes::attachAttached | gputpcgmmergertypes::attachTube; + GPUAtomic(uint32_t)* const weights = merger.ClusterAttachment(); if (goodLeg) { myWeight |= gputpcgmmergertypes::attachGoodLeg; } @@ -671,10 +702,10 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric return uncorrectedY; } -GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) +GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) { static constexpr float kSectAngle = 2 * M_PI / 18.f; - if (Merger->Param().rec.tpc.disableRefitAttachment & 2) { + if (merger.Param().rec.tpc.disableRefitAttachment & 2) { return dodEdx; } if (CAMath::Abs(lastRow - toRow) < 2) { @@ -695,14 +726,14 @@ GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger* GP } if (dodEdx && iRow + step == toRow) { float yUncorrected, zUncorrected; - Merger->GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); + merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, yUncorrected)); - if (pad >= GPUTPCGeometry::NPads(iRow) || (Merger->GetConstantMem()->calibObjects.dEdxCalibContainer && Merger->GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { + if (pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { dodEdx = false; } } CADEBUG(printf("Attaching in row %d\n", iRow)); - AttachClusters(Merger, sector, iRow, iTrack, goodLeg, prop); + AttachClusters(merger, sector, iRow, iTrack, goodLeg, prop); } return dodEdx; } @@ -720,7 +751,7 @@ GPUdii() void GPUTPCGMTrackParam::StoreOuter(gputpcgmmergertypes::GPUTPCOuterPar outerParam->alpha = alpha; } -GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha) +GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha) { if (iRow == 0 || iRow == GPUCA_ROW_COUNT - 1) { return; @@ -735,10 +766,10 @@ GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger* return; } - uint32_t nLoopData = CAMath::AtomicAdd(&Merger->Memory()->nLoopData, 1u); - if (nLoopData >= Merger->NMaxTracks()) { - Merger->raiseError(GPUErrors::ERROR_MERGER_LOOPER_OVERFLOW, nLoopData, Merger->NMaxTracks()); - CAMath::AtomicExch(&Merger->Memory()->nLoopData, Merger->NMaxTracks()); + uint32_t nLoopData = CAMath::AtomicAdd(&merger.Memory()->nLoopData, 1u); + if (nLoopData >= merger.NMaxTracks()) { + merger.raiseError(GPUErrors::ERROR_MERGER_LOOPER_OVERFLOW, nLoopData, merger.NMaxTracks()); + CAMath::AtomicExch(&merger.Memory()->nLoopData, merger.NMaxTracks()); return; } GPUTPCGMLoopData data; @@ -748,36 +779,36 @@ GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger* data.sector = sector; data.row = iRow; data.outwards = outwards; - Merger->LoopData()[nLoopData] = data; + merger.LoopData()[nLoopData] = data; } -GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t loopIdx) +GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t loopIdx) { GPUTPCGMPropagator prop; prop.SetMaterialTPC(); - prop.SetPolynomialField(&Merger->Param().polynomialField); + prop.SetPolynomialField(&merger.Param().polynomialField); prop.SetMaxSinPhi(GPUCA_MAX_SIN_PHI); - prop.SetMatLUT(Merger->Param().rec.useMatLUT ? Merger->GetConstantMem()->calibObjects.matLUT : nullptr); + prop.SetMatLUT(merger.Param().rec.useMatLUT ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); prop.SetSeedingErrors(false); prop.SetFitInProjections(true); prop.SetPropagateBzOnly(false); - GPUTPCGMLoopData& data = Merger->LoopData()[loopIdx]; + GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; prop.SetTrack(&data.param, data.alpha); if (false) { - data.param.AttachClustersLooper(Merger, data.sector, data.row, data.track, data.outwards, prop); + data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); } else { - data.param.AttachClustersLooperFollow(Merger, prop, data.sector, data.row, data.track, data.outwards); + data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.row, data.track, data.outwards); } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger* GPUrestrict() Merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iRow, int32_t iTrack, bool up) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iRow, int32_t iTrack, bool up) { float toX = mX; - bool inFlyDirection = (Merger->MergedTracks()[iTrack].Leg() & 1) ^ up; + bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; static constexpr float kSectAngle = 2 * M_PI / 18.f; - const GPUParam& GPUrestrict() param = Merger->Param(); + const GPUParam& GPUrestrict() param = merger.Param(); bool right = (mP[2] < 0) ^ up; const int32_t sectorSide = sector >= (GPUCA_NSECTORS / 2) ? (GPUCA_NSECTORS / 2) : 0; float lrFactor = right ^ !up ? 1.f : -1.f; @@ -814,7 +845,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger float rowX = GPUTPCGeometry::Row2X(j); if (CAMath::Abs(rowX - (-mP[0] * lrFactor)) < 1.5f) { CADEBUG(printf("\t\tAttempt row %d (X %f Y %f Z %f)\n", j, rowX, mX * lrFactor, mP[1])); - AttachClusters(Merger, sector, j, iTrack, false, mX * lrFactor, mP[1]); + AttachClusters(merger, sector, j, iTrack, false, mX * lrFactor, mP[1]); } } } @@ -840,7 +871,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) { static constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! @@ -890,31 +921,32 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger* GPUr float rowX = mX + GPUTPCGeometry::Row2X(j) - myRowX; if (CAMath::Abs(rowX - paramX) < 1.5f) { // printf("Attempt row %d at y %f\n", j, X); - AttachClusters(Merger, sector, j, iTrack, false, mP[2] > 0 ? X : -X, Z); + AttachClusters(merger, sector, j, iTrack, false, mP[2] > 0 ? X : -X, Z); } } } } -GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, int32_t N) +GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, int32_t N) { if (N == 0) { N = 1; } - const auto& GPUrestrict() cls = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear; + const auto& GPUrestrict() cls = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear; float z0 = cls[clusters[0].num].getTime(), zn = cls[clusters[N - 1].num].getTime(); const auto tmp = zn > z0 ? std::array{zn, z0, GPUTPCGeometry::Row2X(clusters[N - 1].row)} : std::array{z0, zn, GPUTPCGeometry::Row2X(clusters[0].row)}; return ShiftZ(merger, clusters[0].sector, tmp[0], tmp[1], tmp[2]); } -GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger* GPUrestrict() merger, int32_t sector, float cltmax, float cltmin, float clx) +GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, float cltmax, float cltmin, float clx) { - if (!merger->Param().par.continuousTracking) { + const GPUParam& GPUrestrict() param = merger.Param(); + if (!param.par.continuousTracking) { return 0.f; } float deltaZ = 0.f; bool beamlineReached = false; - const float r1 = CAMath::Max(0.0001f, CAMath::Abs(mP[4] * merger->Param().polynomialField.GetNominalBz())); + const float r1 = CAMath::Max(0.0001f, CAMath::Abs(mP[4] * param.polynomialField.GetNominalBz())); if (r1 < 0.01501) { // 100 MeV @ 0.5T ~ 0.66m cutof const float dist2 = mX * mX + mP[0] * mP[0]; const float dist1r2 = dist2 * r1 * r1; @@ -941,16 +973,16 @@ GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger* GPUrestrict() merg } if (!beamlineReached) { - float refZ = ((sector < GPUCA_NSECTORS / 2) ? merger->Param().rec.tpc.defaultZOffsetOverR : -merger->Param().rec.tpc.defaultZOffsetOverR) * clx; + float refZ = ((sector < GPUCA_NSECTORS / 2) ? param.rec.tpc.defaultZOffsetOverR : -param.rec.tpc.defaultZOffsetOverR) * clx; float basez; - merger->GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->TransformIdealZ(sector, cltmax, basez, mTOffset); + merger.GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->TransformIdealZ(sector, cltmax, basez, mTOffset); deltaZ = basez - refZ; } { - float deltaT = merger->GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convDeltaZtoDeltaTimeInTimeFrame(sector, deltaZ); + float deltaT = merger.GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convDeltaZtoDeltaTimeInTimeFrame(sector, deltaZ); mTOffset += deltaT; - const float maxT = cltmin - merger->GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->getT0(); - const float minT = cltmax - merger->GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->getMaxDriftTime(sector); + const float maxT = cltmin - merger.GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->getT0(); + const float minT = cltmax - merger.GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->getMaxDriftTime(sector); // printf("T Check: Clusters %f %f, min %f max %f vtx %f\n", tz1, tz2, minT, maxT, mTOffset); deltaT = 0.f; if (mTOffset < minT) { @@ -960,7 +992,7 @@ GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger* GPUrestrict() merg deltaT = maxT - mTOffset; } if (deltaT != 0.f) { - deltaZ += merger->GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convDeltaTimeToDeltaZinTimeFrame(sector, deltaT); + deltaZ += merger.GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convDeltaTimeToDeltaZinTimeFrame(sector, deltaT); // printf("Moving clusters to TPC Range: QPt %f, New mTOffset %f, t1 %f, t2 %f, Shift %f in Z: %f to %f --> %f to %f in T\n", mP[4], mTOffset + deltaT, tz1, tz2, deltaZ, tz2 - mTOffset, tz1 - mTOffset, tz2 - mTOffset - deltaT, tz1 - mTOffset - deltaT); mTOffset += deltaT; } @@ -1002,7 +1034,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. { if (!track.OK()) { return; @@ -1016,27 +1048,27 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, track, rebuilt); - CADEBUG(if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); + bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt); + CADEBUG(if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (CAMath::Abs(t.QPt()) < 1.e-4f) { t.QPt() = CAMath::Copysign(1.e-4f, t.QPt()); } - CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger->MergedTracks()[iTrk].Looper()); }); + CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger.MergedTracks()[iTrk].Looper()); }); - track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? - if (t.GetNDF() <= 0 && !rebuilt && merger->Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? + track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? + if (t.GetNDF() <= 0 && !rebuilt && merger.Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? track.Param().NDF() = 0; } else { track.Param() = t; track.Alpha() = Alpha; } - if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) { + if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) { track.SetNClustersFitted(nTrackHits); } - // if (track.OK()) merger->DebugRefitMergedTrack(track); + // if (track.OK()) merger.DebugRefitMergedTrack(track); } GPUd() void GPUTPCGMTrackParam::Rotate(float alpha) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 9f2d1b97c6981..88c8abefac8ed 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -27,8 +27,14 @@ class AliExternalTrackParam; +namespace o2::tpc +{ +struct ClusterNative; +}; + namespace o2::gpu { +class GPUTPCTracker; class GPUTPCGMMerger; class GPUTPCGMBorderTrack; struct GPUParam; @@ -36,6 +42,13 @@ class GPUTPCGMPhysicalTrackModel; class GPUTPCGMPolynomialField; class GPUTPCGMMergedTrack; class GPUTPCGMPropagator; +class GPUdEdx; +struct GPUTPCGMMergedTrackHit; + +namespace gputpcgmmergertypes +{ +struct InterpolationErrorHit; +} // namespace gputpcgmmergertypes /** * @class GPUTPCGMTrackParam @@ -141,23 +154,26 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, bool rebuilt); + GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() void FitdEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, const GPUParam& param, bool finalFit, const GPUCalibObjectsConst& calib, int ihit, int ihitMergeFirst, int wayDirection, const o2::tpc::ClusterNative* clustersArray, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); - GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); - - GPUd() bool AttachClustersPropagate(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop, bool inFlyDirection, float maxSinPhi = GPUCA_MAX_SIN_PHI, bool checkdEdx = false); - GPUd() float AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use - GPUd() float AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); - GPUd() void AttachClustersLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& prop); - GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger* GPUrestrict() Merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards); - GPUd() void StoreLoopPropagation(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); + GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); + GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); + + GPUd() bool AttachClustersPropagate(const GPUTPCGMMerger& merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop, bool inFlyDirection, float maxSinPhi = GPUCA_MAX_SIN_PHI, bool checkdEdx = false); + GPUd() float AttachClusters(const GPUTPCGMMerger& merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use + GPUd() float AttachClusters(const GPUTPCGMMerger& merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); + GPUd() void AttachClustersLooper(const GPUTPCGMMerger& merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& prop); + GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards); + GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); GPUd() void StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha); - GPUd() static void PropagateLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t loopIdx); + GPUd() static void PropagateLooper(const GPUTPCGMMerger& merger, int32_t loopIdx); - GPUd() void AddCovDiagErrors(const float* GPUrestrict() errors2); - GPUd() void AddCovDiagErrorsWithCorrelations(const float* GPUrestrict() errors2); + GPUd() void AddCovDiagErrors(const float* errors2); + GPUd() void AddCovDiagErrorsWithCorrelations(const float* errors2); GPUdi() void MarkClusters(GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihitFirst, int32_t ihitLast, int32_t wayDirection, uint8_t state) { @@ -183,8 +199,8 @@ class GPUTPCGMTrackParam } GPUd() void Rotate(float alpha); - GPUd() float ShiftZ(const GPUTPCGMMerger* merger, int32_t sector, float cltmax, float cltmin, float clx); - GPUd() float ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, int32_t N); + GPUd() float ShiftZ(const GPUTPCGMMerger& merger, int32_t sector, float cltmax, float cltmin, float clx); + GPUd() float ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, int32_t N); GPUd() static float Reciprocal(float x) { return 1.f / x; } GPUdi() static void Assign(float& x, bool mask, float v) From 65d12bcabb621dd03b0aefe95f5bd175eba15d1d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 23 Oct 2025 13:25:01 +0200 Subject: [PATCH 08/25] GPU: Add looperFollowMode option --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index c19b32ecc305c..8786d350f6333 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -147,6 +147,7 @@ AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fractio AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following)") +AddOptionRTC(looperFollowMode, uint8_t, 1, "", 0, "0 = simple, 1 = advances, for disabling use disableRefitAttachment = 4") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression (0 = no, 1 = strategy A, 2 = strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits (1=truncate charge/width LSB, 2=differences, 4=track-model)") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 0d4b76d8a7492..a2084565f9692 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -795,10 +795,10 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; prop.SetTrack(&data.param, data.alpha); - if (false) { - data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); - } else { + if (merger.Param().rec.tpc.looperFollowMode == 1) { data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.row, data.track, data.outwards); + } else { + data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); } } From 7f4f8dce18c60695f269812ac1849d839e52844c Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 23 Oct 2025 13:25:20 +0200 Subject: [PATCH 09/25] GPU: simplifaction, refactoring and some minor improvements --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx | 145 ++++++------- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 12 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 196 ++++++++---------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 7 +- 5 files changed, 171 insertions(+), 191 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 8786d350f6333..9e2e7c79f9980 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -141,7 +141,7 @@ AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger ( AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") -AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") +AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx index e91426b51e5c4..6438a66e3b8bc 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx @@ -608,7 +608,7 @@ GPUd() float GPUTPCGMPropagator::PredictChi2(float posY, float posZ, float err2Y } } -GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, int8_t rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge) +GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, bool rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge) { float err2Y, err2Z; GetErr2(err2Y, err2Z, param, posZ, iRow, clusterState, sector, time, avgInvCharge, invCharge); @@ -616,7 +616,7 @@ GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, return Update(posY, posZ, iRow, param, clusterState, rejectChi2, refit, err2Y, err2Z); } -GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, int8_t rejectChi2, bool refit, float err2Y, float err2Z) +GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z) { if (mT->NDF() == -5) { // first measurement: no need to filter, as the result is known in advance. just set it. mT->ResetCovariance(); @@ -635,80 +635,83 @@ GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, return 0; } - return Update(posY, posZ, clusterState, rejectChi2 == rejectDirect, err2Y, err2Z, ¶m); + return Update(posY, posZ, clusterState, rejectChi2, err2Y, err2Z, ¶m); } -GPUd() int32_t GPUTPCGMPropagator::InterpolateReject(const GPUParam& GPUrestrict() param, float posY, float posZ, int16_t clusterState, int8_t rejectChi2, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ) +GPUd() void GPUTPCGMPropagator::InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter) { float* GPUrestrict() mC = mT->Cov(); float* GPUrestrict() mP = mT->Par(); - if (rejectChi2 == rejectInterFill) { - inter->posY = mP[0]; - inter->posZ = mP[1]; - if (mT->NDF() <= 0) { - inter->errorY = inter->errorZ = 100.f; - } else { - inter->errorY = mC[0]; - inter->errorZ = mC[2]; - } - } else if (rejectChi2 == rejectInterReject) { - float chi2Y, chi2Z; - if (mT->NDF() <= 0) { - chi2Y = CAMath::Square((float)inter->posY - posY) / ((float)inter->errorY + err2Y); - chi2Z = CAMath::Square((float)inter->posZ + deltaZ - posZ) / ((float)inter->errorZ + err2Z); - } else if (mFitInProjections) { - const float Iz0 = inter->posY - mP[0]; - const float Iz1 = inter->posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter->errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter->errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; - // printf("\t%21sInterpo ----- abde artaf%16s Y %8.3f, Z %8.3f (Errors %f <-- (%f, %f) %f <-- (%f, %f))\n", "", "", ImP0, ImP1, sqrtf(ImC0), sqrtf(mC[0]), sqrtf(inter->errorY), sqrtf(ImC2), sqrtf(mC[2]), sqrtf(inter->errorZ)); - const float Jz0 = posY - ImP0; - const float Jz1 = posZ - ImP1; - const float Jw0 = 1.f / (ImC0 + err2Y); - const float Jw2 = 1.f / (ImC2 + err2Z); - chi2Y = Jw0 * Jz0 * Jz0; - chi2Z = Jw2 * Jz1 * Jz1; - } else { - const float Iz0 = inter->posY - mP[0]; - const float Iz1 = inter->posZ + deltaZ - mP[1]; - float Iw0 = mC[2] + (float)inter->errorZ; - float Iw2 = mC[0] + (float)inter->errorY; - float Idet = CAMath::Max(1e-10f, Iw0 * Iw2 - mC[1] * mC[1]); - Idet = 1.f / Idet; - Iw0 *= Idet; - const float Iw1 = mC[1] * Idet; - Iw2 *= Idet; - const float Ik00 = mC[0] * Iw0 + mC[1] * Iw1; - const float Ik01 = mC[0] * Iw1 + mC[1] * Iw2; - const float Ik10 = mC[1] * Iw0 + mC[2] * Iw1; - const float Ik11 = mC[1] * Iw1 + mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0 + Ik01 * Iz1; - const float ImP1 = mP[1] + Ik10 * Iz0 + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0] + Ik01 * mC[1]; - const float ImC1 = mC[1] - Ik10 * mC[0] + Ik11 * mC[1]; - const float ImC2 = mC[2] - Ik10 * mC[1] + Ik11 * mC[2]; - const float Jz0 = posY - ImP0; - const float Jz1 = posZ - ImP1; - float Jw0 = ImC2 + err2Z; - float Jw2 = ImC0 + err2Y; - float Jdet = CAMath::Max(1e-10f, Jw0 * Jw2 - ImC1 * ImC1); - Jdet = 1.f / Jdet; - Jw0 *= Jdet; - const float Jw1 = ImC1 * Jdet; - Jw2 *= Jdet; - chi2Y = CAMath::Abs((Jw0 * Jz0 + Jw1 * Jz1) * Jz0); - chi2Z = CAMath::Abs((Jw1 * Jz0 + Jw2 * Jz1) * Jz1); - } - if (RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? - // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); - return updateErrorClusterRejectedInInterpolation; - } + inter->posY = mP[0]; + inter->posZ = mP[1]; + if (mT->NDF() <= 0) { + inter->errorY = inter->errorZ = 100.f; + } else { + inter->errorY = mC[0]; + inter->errorZ = mC[2]; + } +} + +GPUd() int32_t GPUTPCGMPropagator::InterpolateReject(const GPUParam& GPUrestrict() param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ) +{ + float* GPUrestrict() mC = mT->Cov(); + float* GPUrestrict() mP = mT->Par(); + float chi2Y, chi2Z; + if (mT->NDF() <= 0) { + chi2Y = CAMath::Square((float)inter->posY - posY) / ((float)inter->errorY + err2Y); + chi2Z = CAMath::Square((float)inter->posZ + deltaZ - posZ) / ((float)inter->errorZ + err2Z); + } else if (mFitInProjections) { + const float Iz0 = inter->posY - mP[0]; + const float Iz1 = inter->posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter->errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter->errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0; + const float ImP1 = mP[1] + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0]; + const float ImC2 = mC[2] - Ik11 * mC[2]; + // printf("\t%21sInterpo ----- abde artaf%16s Y %8.3f, Z %8.3f (Errors %f <-- (%f, %f) %f <-- (%f, %f))\n", "", "", ImP0, ImP1, sqrtf(ImC0), sqrtf(mC[0]), sqrtf(inter->errorY), sqrtf(ImC2), sqrtf(mC[2]), sqrtf(inter->errorZ)); + const float Jz0 = posY - ImP0; + const float Jz1 = posZ - ImP1; + const float Jw0 = 1.f / (ImC0 + err2Y); + const float Jw2 = 1.f / (ImC2 + err2Z); + chi2Y = Jw0 * Jz0 * Jz0; + chi2Z = Jw2 * Jz1 * Jz1; + } else { + const float Iz0 = inter->posY - mP[0]; + const float Iz1 = inter->posZ + deltaZ - mP[1]; + float Iw0 = mC[2] + (float)inter->errorZ; + float Iw2 = mC[0] + (float)inter->errorY; + float Idet = CAMath::Max(1e-10f, Iw0 * Iw2 - mC[1] * mC[1]); + Idet = 1.f / Idet; + Iw0 *= Idet; + const float Iw1 = mC[1] * Idet; + Iw2 *= Idet; + const float Ik00 = mC[0] * Iw0 + mC[1] * Iw1; + const float Ik01 = mC[0] * Iw1 + mC[1] * Iw2; + const float Ik10 = mC[1] * Iw0 + mC[2] * Iw1; + const float Ik11 = mC[1] * Iw1 + mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0 + Ik01 * Iz1; + const float ImP1 = mP[1] + Ik10 * Iz0 + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0] + Ik01 * mC[1]; + const float ImC1 = mC[1] - Ik10 * mC[0] + Ik11 * mC[1]; + const float ImC2 = mC[2] - Ik10 * mC[1] + Ik11 * mC[2]; + const float Jz0 = posY - ImP0; + const float Jz1 = posZ - ImP1; + float Jw0 = ImC2 + err2Z; + float Jw2 = ImC0 + err2Y; + float Jdet = CAMath::Max(1e-10f, Jw0 * Jw2 - ImC1 * ImC1); + Jdet = 1.f / Jdet; + Jw0 *= Jdet; + const float Jw1 = ImC1 * Jdet; + Jw2 *= Jdet; + chi2Y = CAMath::Abs((Jw0 * Jz0 + Jw1 * Jz1) * Jz0); + chi2Z = CAMath::Abs((Jw1 * Jz0 + Jw2 * Jz1) * Jz1); + } + if (RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? + // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); + return updateErrorClusterRejectedInInterpolation; } return 0; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index 47e6c870dac25..c97b093bd131e 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -57,11 +57,6 @@ class GPUTPCGMPropagator updateErrorClusterRejectedInUpdate = 4, updateErrorClusterRejectedEdge = 5 }; - enum RejectChi2Mode { - rejectDirect = 1, - rejectInterFill = 2, - rejectInterReject = 3 - }; GPUdDefault() GPUTPCGMPropagator() = default; @@ -104,10 +99,11 @@ class GPUTPCGMPropagator GPUd() int32_t PropagateToXAlphaBz(float posX, float posAlpha, bool inFlyDirection); - GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, int8_t rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge); - GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, int8_t rejectChi2, bool refit, float err2Y, float err2Z); + GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge); + GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z); GPUd() int32_t Update(float posY, float posZ, int16_t clusterState, bool rejectChi2, float err2Y, float err2Z, const GPUParam* param = nullptr); - GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, int8_t rejectChi2, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); + GPUd() void InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter); + GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); GPUd() float PredictChi2(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, int8_t sideC, float time, float avgCharge, float charge) const; GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; GPUd() static int32_t RejectCluster(float chiY, float chiZ, uint8_t clusterState) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index a2084565f9692..9805cc696f094 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -51,7 +51,7 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { static constexpr float maxSinPhi = GPUCA_MAX_SIN_PHI; - CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); + static constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = merger.Param(); GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -105,11 +105,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t wayDirection = (iWay & 1) ? -1 : 1; for (int32_t ihit = ihitStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { - const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); - if (crossCE) { - lastSector = clusters[ihit].sector; - } - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { @@ -131,11 +126,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; merger.GetConstantMem()->calibObjects.fastTransformHelper->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); } - // clang-format off CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - // CADEBUG(if ((uint32_t)merger.GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) - // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger.GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); - // clang-format on if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; nMissed2++; @@ -152,33 +143,64 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } const auto& cluster = clusters[ihit]; - - // clang-format off - CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - // clang-format on - if (!param.rec.tpc.rebuildTrackInFit && allowChangeClusters && lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; - dodEdx = AttachClustersPropagate(merger, cluster.sector, lastRow, cluster.row, iTrk, track.Leg() == 0, prop, inFlyDirection, GPUCA_MAX_SIN_PHI, dodEdx); - if (dodEdx) { - dEdx.fillSubThreshold(lastRow - wayDirection); - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - dEdxAlt.fillSubThreshold(lastRow - wayDirection); + if (lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { + bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; + bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); + bool doIntertpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); + if (dodEdx || doAttach || doIntertpolate) { + int32_t step = cluster.row > lastRow ? 1 : -1; + float tmpX, tmpY, tmpZ; + for (int32_t iRow = lastRow + step; iRow != cluster.row; iRow += step) { + if (CAMath::Abs(mP[2]) > maxSinPhi || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(kSectAngle / 2.f)) { + break; + } + if (prop.GetPropagatedYZ(mX - GPUTPCGeometry::Row2X(iRow - step) + GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + break; + } + merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); + if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { + break; + } + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + if (dodEdx) { + float yUncorrected, zUncorrected; + merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(cluster.sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(cluster.sector, iRow, yUncorrected)); + if (!(pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(cluster.sector, iRow, pad)))) { + dEdx.fillSubThreshold(iRow); + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { + dEdxAlt.fillSubThreshold(iRow); + } + } + } + } + if (doAttach) { + AttachClusters(merger, cluster.sector, iRow, iTrk, track.Leg() == 0, prop); } } } } - auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; + CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - int32_t retValHit = FitHit(merger, iTrk, track, N, NTolerated, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, lastRow, lastSector, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, ihitStart, allowChangeClusters, refit, finalFit, nMissed, nMissed2, covYYUpd, resetT0); - if (retValHit == 1) { + auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; + const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0); + if (retValHit == 0) { + ihitStart = ihit; + N++; + covYYUpd = mC[0]; + } else if (retValHit == 1) { break; - } else if (retValHit == -1) { + } else if (retValHit == 2) { + NTolerated++; continue; } + + lastRow = cluster.row; + lastSector = cluster.sector; } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255) { StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastRow %d row %d out %d\n", iTrk, (int)lastRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -222,7 +244,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -239,26 +261,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); } } - if (retValProp == 0) { - lastRow = cluster.row; - lastSector = cluster.sector; - } // clang-format off CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on - if (crossCE) { - if (param.rec.tpc.addErrorsCECrossing) { - if (param.rec.tpc.addErrorsCECrossing >= 2) { - AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); - } else { - AddCovDiagErrors(param.rec.tpc.errorsCECrossing); - } - } else if (mC[2] < 0.5f) { - mC[2] = 0.5f; - } - } - float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); @@ -270,33 +276,33 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - return 1; + return 1; // bad chi2 for the whole track, stop the fit } if (retValProp || sinPhiErr) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; - NTolerated++; CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); - return -1; + return 2; // Propagate failed or high incl angle } CADEBUG(printf("\n")); + if (crossCE) { + if (param.rec.tpc.addErrorsCECrossing) { + if (param.rec.tpc.addErrorsCECrossing >= 2) { + AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); + } else { + AddCovDiagErrors(param.rec.tpc.errorsCECrossing); + } + } else if (mC[2] < 0.5f) { + mC[2] = 0.5f; + } + } + int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; } else { - int8_t rejectChi2 = 0; - if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { - if (iWay == nWays - 3) { - rejectChi2 = GPUTPCGMPropagator::rejectInterFill; - } else if (iWay == nWays - 2) { - rejectChi2 = GPUTPCGMPropagator::rejectInterReject; - } else if (iWay == nWays - 1) { - rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; - } - } - float err2Y, err2Z; const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; const float invSqrtCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; @@ -306,12 +312,23 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { - if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = GPUTPCGMPropagator::rejectDirect; - } else { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); + bool rejectChi2 = false; + if (param.rec.tpc.mergerInterpolateErrors) { + if (iWay == nWays - 3) { + prop.InterpolateFill(yy, zz, &inter); + } else if (iWay == nWays - 2) { + if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + rejectChi2 = true; + } else if (!param.rec.tpc.rebuildTrackInFit) { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } + } else if (iWay == nWays - 1) { + if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { + rejectChi2 = true; + } } + } else { + rejectChi2 = GetNDF() > 0 && allowChangeClusters; } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters @@ -332,11 +349,8 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (!retValUpd && !retValInt) // track is updated { lastUpdateX = mX; - covYYUpd = mC[0]; nMissed = nMissed2 = 0; UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - N++; - ihitStart = ihit; float dy = mP[0] - prop.Model().Y(); float dz = mP[1] - prop.Model().Z(); if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { @@ -344,6 +358,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.SetTrack(this, prop.GetAlpha()); } FitdEdx(dEdx, dEdxAlt, param, finalFit, merger.GetConstantMem()->calibObjects, ihit, ihitMergeFirst, wayDirection, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear, clusters, clusterState, zz); + return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track if (retValInt || allowChangeClusters) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); @@ -354,10 +369,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger nMissed++; nMissed2++; } + return -1; // cluster rejected } else { return 1; // bad chi2 for the whole track, stop the fit } - return 0; } GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) @@ -393,6 +408,9 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); int32_t nCandidates = 0; + while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + nCandidates].id > 1) { + nCandidates++; + } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { const float kFactor = tracker.GetChiSeedFactor(); const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? @@ -702,42 +720,6 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric return uncorrectedY; } -GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) -{ - static constexpr float kSectAngle = 2 * M_PI / 18.f; - if (merger.Param().rec.tpc.disableRefitAttachment & 2) { - return dodEdx; - } - if (CAMath::Abs(lastRow - toRow) < 2) { - return dodEdx; - } - int32_t step = toRow > lastRow ? 1 : -1; - float xx = mX - GPUTPCGeometry::Row2X(lastRow); - for (int32_t iRow = lastRow + step; iRow != toRow; iRow += step) { - if (CAMath::Abs(mP[2]) > maxSinPhi) { - return dodEdx; - } - if (CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(kSectAngle / 2.f)) { - return dodEdx; - } - int32_t err = prop.PropagateToXAlpha(xx + GPUTPCGeometry::Row2X(iRow), prop.GetAlpha(), inFlyDirection); - if (err) { - return dodEdx; - } - if (dodEdx && iRow + step == toRow) { - float yUncorrected, zUncorrected; - merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); - uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, yUncorrected)); - if (pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { - dodEdx = false; - } - } - CADEBUG(printf("Attaching in row %d\n", iRow)); - AttachClusters(merger, sector, iRow, iTrack, goodLeg, prop); - } - return dodEdx; -} - GPUdii() void GPUTPCGMTrackParam::StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha) { CADEBUG(printf("\t%21sStorO Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f, SP %5.2f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f\n", "", alpha, mX, mP[0], mP[1], mP[4], mP[2], sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]))); @@ -802,7 +784,7 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iRow, int32_t iTrack, bool up) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up) { float toX = mX; bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; @@ -871,7 +853,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& GPUrestrict() prop) { static constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! @@ -881,7 +863,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr float SinPhi = CAMath::Sqrt(1 - mP[2] * mP[2]) * (mP[2] > 0 ? -1 : 1); float b = prop.GetBz(prop.GetAlpha(), mX, mP[0], mP[1]); - float dx = outwards ? 1.f : -1.f; + float dx = up ? 1.f : -1.f; const float myRowX = GPUTPCGeometry::Row2X(iRow); // printf("\nAttachMirror sector %d row %d outwards %d\n", (int)sector, (int)iRow, (int)outwards); // printf("X %f Y %f Z %f SinPhi %f -->\n", mX, mP[0], mP[1], mP[2]); @@ -915,7 +897,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr // printf("count %d: At X %f Y %f Z %f SinPhi %f\n", maxTries, mP[2] > 0 ? -Y : Y, mP[2] > 0 ? X : -X, Z, SinPhi); float paramX = mP[2] > 0 ? -Y : Y; - int32_t step = outwards ? 1 : -1; + int32_t step = up ? 1 : -1; int32_t found = 0; for (int32_t j = iRow; j >= 0 && j < GPUCA_ROW_COUNT && found < 3; j += step) { float rowX = mX + GPUTPCGeometry::Row2X(j) - myRowX; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 88c8abefac8ed..60a03f522d2d2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,18 +156,17 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void FitdEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, const GPUParam& param, bool finalFit, const GPUCalibObjectsConst& calib, int ihit, int ihitMergeFirst, int wayDirection, const o2::tpc::ClusterNative* clustersArray, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); - GPUd() bool AttachClustersPropagate(const GPUTPCGMMerger& merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop, bool inFlyDirection, float maxSinPhi = GPUCA_MAX_SIN_PHI, bool checkdEdx = false); GPUd() float AttachClusters(const GPUTPCGMMerger& merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use GPUd() float AttachClusters(const GPUTPCGMMerger& merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); - GPUd() void AttachClustersLooper(const GPUTPCGMMerger& merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& prop); - GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards); + GPUd() void AttachClustersLooper(const GPUTPCGMMerger& merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& prop); + GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up); GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); GPUd() void StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha); GPUd() static void PropagateLooper(const GPUTPCGMMerger& merger, int32_t loopIdx); From 6d295be9bb786dabc09bbe5027a0e749ce58aba5 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 27 Oct 2025 13:34:27 +0100 Subject: [PATCH 10/25] GPU TPC: Refactor cluster merging --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 98 +++++++++++-------- 1 file changed, 58 insertions(+), 40 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 9805cc696f094..116460ce6c960 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -122,26 +122,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; - { - const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; - merger.GetConstantMem()->calibObjects.fastTransformHelper->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); - } CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { + if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters) == -1) { nMissed++; nMissed2++; continue; } - if (param.rec.tpc.rejectIFCLowRadiusCluster) { - const float r2 = xx * xx + yy * yy; - const float rmax = (83.5f + param.rec.tpc.sysClusErrorMinDist); - if (r2 < rmax * rmax) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); - continue; - } - } - const auto& cluster = clusters[ihit]; if (lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; @@ -570,55 +557,86 @@ GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) { + const int32_t ihitFirst = ihit; + { + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + merger.GetConstantMem()->calibObjects.fastTransformHelper->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + } if (ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector) { - float maxDistY, maxDistZ; - prop.GetErr2(maxDistY, maxDistZ, merger.Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge - maxDistY = (maxDistY + mC[0]) * 20.f; - maxDistZ = (maxDistZ + mC[2]) * 20.f; - int32_t noReject = 0; // Cannot reject if simple estimation of y/z fails (extremely unlike case) + float maxDistY2, maxDistZ2; + bool noReject = false; // Cannot reject if simple estimation of y/z fails (extremely unlike case) if (CAMath::Abs(clAlpha - prop.GetAlpha()) > 1.e-4f) { noReject = prop.RotateToAlpha(clAlpha); } float projY = 0, projZ = 0; - if (noReject == 0) { + if (!noReject) { noReject |= prop.GetPropagatedYZ(xx, projY, projZ); } - float count = 0.f; - xx = yy = zz = 0.f; - clusterState = 0; - while (true) { - const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; - float clamp = cl.qTot; - float clx, cly, clz; - merger.GetConstantMem()->calibObjects.fastTransformHelper->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); - float dy = cly - projY; - float dz = clz - projZ; - if (noReject == 0 && (dy * dy > maxDistY || dz * dz > maxDistZ)) { - CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY), sqrtf(maxDistZ), mP[0], projY, cly, mP[1], projZ, clz)); + prop.GetErr2(maxDistY2, maxDistZ2, merger.Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge + const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; + maxDistY2 = (maxDistY2 + mC[0]) * kFactor; + maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; + auto chkFunction = [clusters, rejectChi2, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { + float dy = y - projY; + float dz = z - projZ; + if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { + CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); if (rejectChi2) { - clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; + clusters[ih].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; } + return false; } else { - CADEBUG(printf("\t\tMerging hit row %d X %f Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ihit].row, clx, cly, clz, dy, dz, sqrtf(maxDistY), sqrtf(maxDistZ))); - xx += clx * clamp; // TODO: Weight in pad/time instead of XYZ + CADEBUG(printf("\t\tMerging hit row %d Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ih].row, y, z, dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2))); + return true; + } + }; + const float tmpX = xx; + float count; + if (chkFunction(ihit, yy, zz)) { + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + const float clamp = cl.qTot; + xx *= clamp; + yy *= clamp; + zz *= clamp; + clusterState = clusters[ihit].state; + count = clamp; + } else { + xx = yy = zz = count = 0.f; + clusterState = 0; + } + do { + ihit += wayDirection; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + const float clamp = cl.qTot; + float clx, cly, clz; + merger.GetConstantMem()->calibObjects.fastTransformHelper->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); + if (chkFunction(ihit, cly, clz)) { + xx += clx * clamp; yy += cly * clamp; zz += clz * clamp; clusterState |= clusters[ihit].state; count += clamp; } - if (!(ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector)) { - break; - } - ihit += wayDirection; - } + } while (ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector); if (count < 0.1f) { CADEBUG(printf("\t\tNo matching cluster in double-row, skipping\n")); + xx = tmpX; return -1; } xx /= count; yy /= count; zz /= count; } + if (merger.Param().rec.tpc.rejectIFCLowRadiusCluster) { + const float r2 = xx * xx + yy * yy; + const float rmax2 = CAMath::Square(83.5f + merger.Param().rec.tpc.sysClusErrorMinDist); + if (r2 < rmax2) { + if (rejectChi2) { + MarkClusters(clusters, ihitFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + } + return -1; + } + } return 0; } From 8dff506960f74744dba26f9a3ffa5f5b40bfb52d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 24 Oct 2025 11:01:17 +0200 Subject: [PATCH 11/25] GPU: Change indexing of interpolated hit temporary data --- GPU/GPUTracking/DataTypes/GPUTPCGeometry.h | 2 + GPU/GPUTracking/Global/GPUErrorCodes.h | 35 ++-- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 1 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 173 ++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 5 +- GPU/GPUTracking/Refit/GPUTrackingRefit.cxx | 5 +- 6 files changed, 125 insertions(+), 96 deletions(-) diff --git a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h index 9ad83bff363ac..2d2697d0da91e 100644 --- a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h +++ b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h @@ -144,6 +144,8 @@ class GPUTPCGeometry // TODO: Make values constexpr const float v = (sector >= GPUCA_NSECTORS / 2) ? -z : z; return (250.f - v) * FACTOR_Z2T; // Used in compression, must remain constant at 250cm } + + GPUd() static constexpr float kSectAngle() { return 2 * M_PI / 18.f; } }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Global/GPUErrorCodes.h b/GPU/GPUTracking/Global/GPUErrorCodes.h index 2080548c7775c..b3894cf267c17 100644 --- a/GPU/GPUTracking/Global/GPUErrorCodes.h +++ b/GPU/GPUTracking/Global/GPUErrorCodes.h @@ -35,22 +35,23 @@ GPUCA_ERROR_CODE(15, ERROR_SECTORDATA_Z_OVERFLOW, Sector, Value) GPUCA_ERROR_CODE(16, ERROR_MERGER_HIT_OVERFLOW, Value, Max) GPUCA_ERROR_CODE(17, ERROR_MERGER_REBUILD_HIT_OVERFLOW, Value, Max) GPUCA_ERROR_CODE(18, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(19, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(20, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(21, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(22, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(23, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(24, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(25, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER -GPUCA_ERROR_CODE(26, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped -GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded -GPUCA_ERROR_CODE(28, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete -GPUCA_ERROR_CODE(29, ERROR_TPCZS_INVALID_OFFSET, SectorEndpoint, Value, Expected) // Raw page is skipped since it contains invalid payload offset -GPUCA_ERROR_CODE(30, ERROR_TPCZS_INVALID_MAGIC_WORD, Value) // ZS header contains wrong magic word -GPUCA_ERROR_CODE(31, ERROR_TPCZS_PAGE_OVERFLOW, Position, PageEnd) // Ran out of page to decode -GPUCA_ERROR_CODE(32, ERROR_TPCZS_VERSION_MISMATCH, Value, Expected) // ZS decoder received page with wrong version -GPUCA_ERROR_CODE(33, ERROR_TPCZS_UNKNOWN, ErrorCode) // Unkown or invalid error code raised in decoder -GPUCA_ERROR_CODE(33, MAX_GPUCA_ERROR_NUMBER) +GPUCA_ERROR_CODE(19, ERROR_MERGER_INTERPOLATION_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(20, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(21, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(22, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(23, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(24, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(25, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(26, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(26, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER +GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped +GPUCA_ERROR_CODE(28, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded +GPUCA_ERROR_CODE(29, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete +GPUCA_ERROR_CODE(30, ERROR_TPCZS_INVALID_OFFSET, SectorEndpoint, Value, Expected) // Raw page is skipped since it contains invalid payload offset +GPUCA_ERROR_CODE(31, ERROR_TPCZS_INVALID_MAGIC_WORD, Value) // ZS header contains wrong magic word +GPUCA_ERROR_CODE(32, ERROR_TPCZS_PAGE_OVERFLOW, Position, PageEnd) // Ran out of page to decode +GPUCA_ERROR_CODE(33, ERROR_TPCZS_VERSION_MISMATCH, Value, Expected) // ZS decoder received page with wrong version +GPUCA_ERROR_CODE(34, ERROR_TPCZS_UNKNOWN, ErrorCode) // Unkown or invalid error code raised in decoder +GPUCA_ERROR_CODE(34, MAX_GPUCA_ERROR_NUMBER) // #define GPUCA_CHECK_TPCZS_CORRUPTION diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index 731439aab0be2..c34e5384e3cef 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -38,6 +38,7 @@ struct InterpolationErrorHit { struct InterpolationErrors { InterpolationErrorHit hit[GPUCA_MERGER_MAX_TRACK_CLUSTERS]; + static constexpr size_t size = GPUCA_MERGER_MAX_TRACK_CLUSTERS; }; struct GPUResolveSharedMemory : public GPUKernelTemplate::GPUSharedMemoryScan64 { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 116460ce6c960..3162517301bef 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -51,7 +51,6 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { static constexpr float maxSinPhi = GPUCA_MAX_SIN_PHI; - static constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = merger.Param(); GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -63,7 +62,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { - for (int32_t i = 0; i < N; i++) { + for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing size interpolation.hit[i].errorY = -1; } } @@ -71,6 +70,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t nWays = param.rec.tpc.nWays; const int32_t maxN = N; int32_t ihitStart = 0; + int32_t interpolatedStart = 0; float covYYUpd = 0.f; float deltaZ = 0.f; @@ -95,86 +95,81 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : GPUCA_MAX_SIN_PHI_LOW); - CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / kSectAngle) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); + CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = 0; - uint8_t lastRow = 255; - uint8_t lastSector = 255; + uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastUpdateSector = 255; float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - for (int32_t ihit = ihitStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { - CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); - if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { - NTolerated++; - } - if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { - clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; + for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart - wayDirection; ihit >= 0 && ihit < maxN; ihit += wayDirection) { + if (!param.rec.tpc.rebuildTrackInFit || rebuilt) { + if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { + CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); + if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { + NTolerated++; + } + if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { + clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; + } + continue; } - continue; } const bool allowChangeClusters = finalOutInFit && (nWays == 1 || ((iWay & 1) ? (ihit <= CAMath::Max(maxN / 2, maxN - 30)) : (ihit >= CAMath::Min(maxN / 2, 30)))); int32_t ihitMergeFirst = ihit; + interpolationIndex += wayDirection; uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters) == -1) { - nMissed++; - nMissed2++; - continue; - } + const int32_t currentClusterStatus = MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters); const auto& cluster = clusters[ihit]; - if (lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; + uint8_t dEdxSubThresholdRow = 255; + bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); + if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { + bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastUpdateSector && currentClusterStatus == 0; bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); - bool doIntertpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); - if (dodEdx || doAttach || doIntertpolate) { - int32_t step = cluster.row > lastRow ? 1 : -1; - float tmpX, tmpY, tmpZ; - for (int32_t iRow = lastRow + step; iRow != cluster.row; iRow += step) { - if (CAMath::Abs(mP[2]) > maxSinPhi || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(kSectAngle / 2.f)) { - break; - } + if (dodEdx || doAttach || doInterpolate) { + int32_t step = cluster.row > lastPropagateRow ? 1 : -1; + for (int32_t iRow = lastPropagateRow + step; iRow != cluster.row; iRow += step) { + float tmpX, tmpY, tmpZ; if (prop.GetPropagatedYZ(mX - GPUTPCGeometry::Row2X(iRow - step) + GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { break; } merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); - if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { - break; - } - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { - if (dodEdx) { - float yUncorrected, zUncorrected; - merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(cluster.sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); - uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(cluster.sector, iRow, yUncorrected)); - if (!(pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(cluster.sector, iRow, pad)))) { - dEdx.fillSubThreshold(iRow); - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - dEdxAlt.fillSubThreshold(iRow); - } - } - } - } - if (doAttach) { - AttachClusters(merger, cluster.sector, iRow, iTrk, track.Leg() == 0, prop); - } + FitAddRow(iRow, cluster.sector, iTrk, track, tmpX, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); } } + interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; + } + lastPropagateRow = cluster.row; + + if (currentClusterStatus) { + nMissed++; + nMissed2++; + if (doInterpolate) { + FitAddRow(cluster.row, cluster.sector, iTrk, track, xx, prop, inFlyDirection, merger, nullptr, false, false, true); + } + continue; } CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; - const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); + if ((uint32_t)interpolationIndex >= interpolation.size) { + merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); + break; + } + auto& inter = interpolation.hit[interpolationIndex]; + const bool crossCE = lastUpdateSector != 255 && ((lastUpdateSector < 18) ^ (clusters[ihit].sector < 18)); int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0); if (retValHit == 0) { + DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; + interpolatedStart = interpolationIndex; N++; covYYUpd = mC[0]; } else if (retValHit == 1) { @@ -184,12 +179,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - lastRow = cluster.row; - lastSector = cluster.sector; + lastUpdateRow = cluster.row; + lastUpdateSector = cluster.sector; + assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255) { - StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); - CADEBUG(printf("\t\tSTORING %d lastRow %d row %d out %d\n", iTrk, (int)lastRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255) { + StoreLoopPropagation(merger, lastUpdateSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); + CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { deltaZ = ShiftZ(clusters, merger, maxN); @@ -231,7 +227,31 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0) +GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float tmpX, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) +{ + if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { + return; + } + if (CAMath::Abs(mP[2]) > GPUCA_MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { + return; + } + const GPUParam& GPUrestrict() param = merger.Param(); + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + if (dodEdx) { + float yUncorrected, zUncorrected; + merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, yUncorrected)); + if (!(pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad)))) { + *dEdxSubThresholdRow = iRow; + } + } + } + if (doAttach) { + AttachClusters(merger, sector, iRow, iTrk, track.Leg() == 0, prop); + } +} + +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -344,8 +364,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger CADEBUG(printf("Reinit linearization\n")); prop.SetTrack(this, prop.GetAlpha()); } - FitdEdx(dEdx, dEdxAlt, param, finalFit, merger.GetConstantMem()->calibObjects, ihit, ihitMergeFirst, wayDirection, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear, clusters, clusterState, zz); - return 0; // ok + return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track if (retValInt || allowChangeClusters) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); @@ -465,9 +484,12 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre return uncorrectedY; } -GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, const GPUParam& GPUrestrict() param, bool finalFit, const GPUCalibObjectsConst& GPUrestrict() calib, int ihit, int ihitMergeFirst, int wayDirection, const ClusterNative* GPUrestrict() clustersArray, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz) +GPUdii() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) { + const GPUParam& GPUrestrict() param = merger.Param(); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + const GPUCalibObjectsConst& GPUrestrict() calib = merger.GetConstantMem()->calibObjects; + const ClusterNative* GPUrestrict() clustersArray = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear; if (param.dodEdxEnabled && finalFit) { // TODO: Costimize flag to remove, and option to remove double-clusters bool acc = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMask) == 0, accAlt = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMaskAlt) == 0; if (acc || accAlt) { @@ -487,10 +509,16 @@ GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& const auto& cluster = clusters[ihit]; if (acc) { dEdx.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime); + if (dEdxSubThresholdRow) { + dEdx.fillSubThreshold(dEdxSubThresholdRow); + } } if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { if (accAlt) { dEdxAlt.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime); + if (dEdxSubThresholdRow) { + dEdxAlt.fillSubThreshold(dEdxSubThresholdRow); + } } } } @@ -498,37 +526,36 @@ GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& } } -GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& Alpha) +GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& GPUrestrict() prop, const GPUParam& GPUrestrict() param, float& GPUrestrict() Alpha) { static constexpr float kDeg2Rad = M_PI / 180.f; - static constexpr float kSectAngle = 2 * M_PI / 18.f; if (param.rec.tpc.trackReferenceX <= 500) { GPUTPCGMTrackParam save = *this; float saveAlpha = Alpha; for (int32_t attempt = 0; attempt < 3; attempt++) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * kSectAngle; + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); Alpha += dAngle; if (prop.PropagateToXAlpha(param.rec.tpc.trackReferenceX, Alpha, 0)) { break; } ConstrainSinPhi(); - if (CAMath::Abs(mP[0]) <= mX * CAMath::Tan(kSectAngle / 2.f)) { + if (CAMath::Abs(mP[0]) <= mX * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { return; } } *this = save; Alpha = saveAlpha; } - if (CAMath::Abs(mP[0]) > mX * CAMath::Tan(kSectAngle / 2.f)) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * kSectAngle; + if (CAMath::Abs(mP[0]) > mX * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); Rotate(dAngle); ConstrainSinPhi(); Alpha += dAngle; } } -GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector) +GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, float toY, float toZ, bool inFlyDirection, const GPUParam& GPUrestrict() param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector) { if (mirrorParameters) { prop.Mirror(inFlyDirection); @@ -555,7 +582,7 @@ GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, mChi2 = 0; } -GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) +GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, const bool markReject) { const int32_t ihitFirst = ihit; { @@ -576,12 +603,12 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; maxDistY2 = (maxDistY2 + mC[0]) * kFactor; maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; - auto chkFunction = [clusters, rejectChi2, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { + auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { float dy = y - projY; float dz = z - projZ; if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); - if (rejectChi2) { + if (markReject) { clusters[ih].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; } return false; @@ -631,7 +658,7 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float r2 = xx * xx + yy * yy; const float rmax2 = CAMath::Square(83.5f + merger.Param().rec.tpc.sysClusErrorMinDist); if (r2 < rmax2) { - if (rejectChi2) { + if (markReject) { MarkClusters(clusters, ihitFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); } return -1; @@ -807,7 +834,6 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger float toX = mX; bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; - static constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = merger.Param(); bool right = (mP[2] < 0) ^ up; const int32_t sectorSide = sector >= (GPUCA_NSECTORS / 2) ? (GPUCA_NSECTORS / 2) : 0; @@ -822,7 +848,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger CADEBUG(printf("\tRotated: X %f Y %f Z %f SinPhi %f (Alpha %f / %f)\n", mP[0], mX, mP[1], mP[2], prop.GetAlpha(), prop.GetAlpha() + CAMath::Pi() / 2.f)); uint32_t maxTries = 100; while (true) { - while (CAMath::Abs(mX) <= CAMath::Abs(mP[0]) * CAMath::Tan(kSectAngle / 2.f) + 0.1f) { + while (CAMath::Abs(mX) <= CAMath::Abs(mP[0]) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f) + 0.1f) { if (maxTries-- == 0) { return; } @@ -873,7 +899,6 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& GPUrestrict() prop) { - static constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! float X = mP[2] > 0 ? mP[0] : -mP[0]; float Y = mP[2] > 0 ? -mX : mX; @@ -908,7 +933,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr Y += dS * SinPhi + h4; Z += dS * mP[3]; SinPhi = newSinPhi; - if (CAMath::Abs(X) > CAMath::Abs(Y) * CAMath::Tan(kSectAngle / 2.f)) { + if (CAMath::Abs(X) > CAMath::Abs(Y) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { // printf("Abort, sector edge\n"); return; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 60a03f522d2d2..0985446b64ee0 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -155,12 +155,13 @@ class GPUTPCGMTrackParam GPUd() bool CheckCov() const; GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); - GPUd() void FitdEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, const GPUParam& param, bool finalFit, const GPUCalibObjectsConst& calib, int ihit, int ihitMergeFirst, int wayDirection, const o2::tpc::ClusterNative* clustersArray, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz); + GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0); + GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float tmpX, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); - GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); + GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); GPUd() float AttachClusters(const GPUTPCGMMerger& merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use diff --git a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx index 4f9d848f2b703..1f7ef61163c5b 100644 --- a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx +++ b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx @@ -399,11 +399,10 @@ GPUd() int32_t GPUTrackingRefit::RefitTrack(T& trkX, bool outward, bool resetCov prop.SetAlpha(alpha); } else if constexpr (std::is_same_v) { static constexpr float kDeg2Rad = M_PI / 180.f; - static constexpr float kSectAngle = 2 * M_PI / 18.f; if (mPparam->rec.tpc.trackReferenceX <= 500) { if (prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX)) { - if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(kSectAngle / 2.f)) { - float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / kDeg2Rad / 20.f) * kSectAngle; + if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { + float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); GPUTPCGMTrackParam::NormalizeAlpha(newAlpha); trk.rotate(newAlpha) && prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX); } From 3264c4670d68c8ec17ea5fbd13e90c9b1dac02e5 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 27 Oct 2025 21:35:02 +0100 Subject: [PATCH 12/25] GPU: Move some functionality to CAMath --- GPU/Common/GPUCommonMath.h | 1 + GPU/GPUTracking/DataTypes/GPUTPCGeometry.h | 3 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 9 +++------ GPU/GPUTracking/Refit/GPUTrackingRefit.cxx | 3 +-- GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h | 2 +- GPU/GPUTracking/qa/GPUQA.cxx | 12 ++++++------ GPU/GPUTracking/qa/genEvents.cxx | 4 ++-- GPU/GPUTracking/qa/genEvents.h | 4 ++-- 8 files changed, 18 insertions(+), 20 deletions(-) diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index 0ff31899dec0c..197820b129bf4 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -95,6 +95,7 @@ class GPUCommonMath GPUhdni() constexpr static float Copysign(float x, float y) { return GPUCA_CHOICE(std::copysignf(x, y), copysignf(x, y), copysign(x, y)); } GPUd() constexpr static float TwoPi() { return 6.2831853f; } GPUd() constexpr static float Pi() { return 3.1415927f; } + GPUd() constexpr static float Deg2Rad() { return Pi() / 180.f; } GPUd() constexpr static float Round(float x); GPUd() constexpr static float Floor(float x) { return GPUCA_CHOICE(floorf(x), floorf(x), floor(x)); } GPUd() static uint32_t Float2UIntReint(const float& x); diff --git a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h index 2d2697d0da91e..68cff65ead39d 100644 --- a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h +++ b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h @@ -16,6 +16,7 @@ #define GPUTPCGEOMETRY_H #include "GPUCommonDef.h" +#include "GPUCommonMath.h" #if !defined(GPUCA_NSECTORS) && !defined(GPUCA_ROW_COUNT) #include "DataFormatsTPC/Constants.h" @@ -145,7 +146,7 @@ class GPUTPCGeometry // TODO: Make values constexpr return (250.f - v) * FACTOR_Z2T; // Used in compression, must remain constant at 250cm } - GPUd() static constexpr float kSectAngle() { return 2 * M_PI / 18.f; } + GPUd() static constexpr float kSectAngle() { return 2 * CAMath::Pi() / 18.f; } }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 3162517301bef..63a15d624a5b2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -279,8 +279,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } - static constexpr float kDeg2Rad = M_PI / 180.f; - const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); + const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { return 1; // bad chi2 for the whole track, stop the fit @@ -528,13 +527,11 @@ GPUdii() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& G GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& GPUrestrict() prop, const GPUParam& GPUrestrict() param, float& GPUrestrict() Alpha) { - static constexpr float kDeg2Rad = M_PI / 180.f; - if (param.rec.tpc.trackReferenceX <= 500) { GPUTPCGMTrackParam save = *this; float saveAlpha = Alpha; for (int32_t attempt = 0; attempt < 3; attempt++) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / CAMath::Deg2Rad() / 20.f) * GPUTPCGeometry::kSectAngle(); Alpha += dAngle; if (prop.PropagateToXAlpha(param.rec.tpc.trackReferenceX, Alpha, 0)) { break; @@ -548,7 +545,7 @@ GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& GPUrestric Alpha = saveAlpha; } if (CAMath::Abs(mP[0]) > mX * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / CAMath::Deg2Rad() / 20.f) * GPUTPCGeometry::kSectAngle(); Rotate(dAngle); ConstrainSinPhi(); Alpha += dAngle; diff --git a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx index 1f7ef61163c5b..be93209082c22 100644 --- a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx +++ b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx @@ -398,11 +398,10 @@ GPUd() int32_t GPUTrackingRefit::RefitTrack(T& trkX, bool outward, bool resetCov trk.NormalizeAlpha(alpha); prop.SetAlpha(alpha); } else if constexpr (std::is_same_v) { - static constexpr float kDeg2Rad = M_PI / 180.f; if (mPparam->rec.tpc.trackReferenceX <= 500) { if (prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX)) { if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { - float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); + float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / CAMath::Deg2Rad() / 20.f) * GPUTPCGeometry::kSectAngle(); GPUTPCGMTrackParam::NormalizeAlpha(newAlpha); trk.rotate(newAlpha) && prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX); } diff --git a/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h b/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h index d45959f99b056..7b7fb9c1de6b1 100644 --- a/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h +++ b/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h @@ -147,7 +147,7 @@ class trackInterface : public GPUTPCGMTrackParam GPUd() float getSnp() const { return GetSinPhi(); } GPUd() float getTgl() const { return GetDzDs(); } GPUd() float getQ2Pt() const { return GetQPt(); } - GPUd() float getEta() const { return -CAMath::Log(CAMath::Tan(0.5f * (0.5f * M_PI - CAMath::ATan(getTgl())))); } + GPUd() float getEta() const { return -CAMath::Log(CAMath::Tan(0.5f * (0.5f * CAMath::Pi() - CAMath::ATan(getTgl())))); } GPUd() float getPt() const { return CAMath::Abs(getQ2Pt()) > 0 ? CAMath::Abs(1.f / getQ2Pt()) : 99999.f; } GPUd() float getSigmaY2() const { return GetErr2Y(); } GPUd() float getSigmaZ2() const { return GetErr2Z(); } diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index e52040e714246..6f29fa7d4efb3 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -183,7 +183,7 @@ static const constexpr int32_t COLORS_HEX[COLORCOUNT] = {0xB03030, 0x00A000, 0x0 static const constexpr int32_t CONFIG_DASHED_MARKERS = 0; static const constexpr float AXES_MIN[5] = {-Y_MAX, -Z_MAX, 0.f, -ETA_MAX, PT_MIN}; -static const constexpr float AXES_MAX[5] = {Y_MAX, Z_MAX, 2.f * M_PI, ETA_MAX, PT_MAX}; +static const constexpr float AXES_MAX[5] = {Y_MAX, Z_MAX, 2.f * CAMath::Pi(), ETA_MAX, PT_MAX}; static const constexpr int32_t AXIS_BINS[5] = {51, 51, 144, 31, 50}; static const constexpr int32_t RES_AXIS_BINS[] = {1017, 113}; // Consecutive bin sizes, histograms are binned down until the maximum entry is 50, each bin size should evenly divide its predecessor. static const constexpr float RES_AXES[5] = {1., 1., 0.03, 0.03, 1.0}; @@ -1214,12 +1214,12 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx const mcInfo_t& info = GetMCTrack(i, iCol); additionalMCParameters& mc2 = mMCParam[iCol][i]; mc2.pt = std::sqrt(info.pX * info.pX + info.pY * info.pY); - mc2.phi = M_PI + std::atan2(-info.pY, -info.pX); + mc2.phi = CAMath::Pi() + std::atan2(-info.pY, -info.pX); float p = info.pX * info.pX + info.pY * info.pY + info.pZ * info.pZ; if (p < 1e-18) { mc2.theta = mc2.eta = 0.f; } else { - mc2.theta = info.pZ == 0 ? (M_PI / 2) : (std::acos(info.pZ / std::sqrt(p))); + mc2.theta = info.pZ == 0 ? (CAMath::Pi() / 2) : (std::acos(info.pZ / std::sqrt(p))); mc2.eta = -std::log(std::tan(0.5 * mc2.theta)); } if (mConfig.writeMCLabels) { @@ -1273,10 +1273,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } float alpha = std::atan2(info.y, info.x); - alpha /= M_PI / 9.f; + alpha /= CAMath::Pi() / 9.f; alpha = std::floor(alpha); - alpha *= M_PI / 9.f; - alpha += M_PI / 18.f; + alpha *= CAMath::Pi() / 9.f; + alpha += CAMath::Pi() / 18.f; float c = std::cos(alpha); float s = std::sin(alpha); diff --git a/GPU/GPUTracking/qa/genEvents.cxx b/GPU/GPUTracking/qa/genEvents.cxx index 6832e48fe17da..b14f873f351c1 100644 --- a/GPU/GPUTracking/qa/genEvents.cxx +++ b/GPU/GPUTracking/qa/genEvents.cxx @@ -200,8 +200,8 @@ int32_t genEvents::GenerateEvent(const GPUParam& param, const char* filename) double eta = gRandom->Uniform(-1.5, 1.5); double theta = 2 * std::atan(1. / exp(eta)); - double lambda = theta - M_PI / 2; - // double theta = gRandom->Uniform(-60,60)*M_PI/180.; + double lambda = theta - CAMath::Pi() / 2; + // double theta = gRandom->Uniform(-60,60)*CAMath::Pi()/180.; double pt = .08 * std::pow(10, gRandom->Uniform(0, 2.2)); double q = 1.; diff --git a/GPU/GPUTracking/qa/genEvents.h b/GPU/GPUTracking/qa/genEvents.h index ee5510c729525..2efe7a983a320 100644 --- a/GPU/GPUTracking/qa/genEvents.h +++ b/GPU/GPUTracking/qa/genEvents.h @@ -16,7 +16,7 @@ #define GENEVENTS_H #include "GPUCommonDef.h" -#include +#include "GPUCommonMath.h" namespace o2::gpu { @@ -66,7 +66,7 @@ class genEvents uint32_t id; }; - const double mTwoPi = 2 * M_PI; + const double mTwoPi = 2 * CAMath::Pi(); const double mSectorDAngle = mTwoPi / 18.; const double mSectorAngleOffset = mSectorDAngle / 2; From b135e8ad8015a7b6a3cbc8a91bfcdfc160736c7a Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 00:11:27 +0100 Subject: [PATCH 13/25] GPU TPC: Store track position for interpolation also when not updating with new hit --- GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx | 2 +- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 133 ++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 4 +- 4 files changed, 76 insertions(+), 65 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx index 6438a66e3b8bc..aef1fd5385290 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx @@ -638,7 +638,7 @@ GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, return Update(posY, posZ, clusterState, rejectChi2, err2Y, err2Z, ¶m); } -GPUd() void GPUTPCGMPropagator::InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter) +GPUd() void GPUTPCGMPropagator::InterpolateFill(gputpcgmmergertypes::InterpolationErrorHit* inter) { float* GPUrestrict() mC = mT->Cov(); float* GPUrestrict() mP = mT->Par(); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index c97b093bd131e..be1dbabee2ae4 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -102,7 +102,7 @@ class GPUTPCGMPropagator GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge); GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z); GPUd() int32_t Update(float posY, float posZ, int16_t clusterState, bool rejectChi2, float err2Y, float err2Z, const GPUParam* param = nullptr); - GPUd() void InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter); + GPUd() void InterpolateFill(gputpcgmmergertypes::InterpolationErrorHit* inter); GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); GPUd() float PredictChi2(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, int8_t sideC, float time, float avgCharge, float charge) const; GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 63a15d624a5b2..a3749c9c893b2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -124,10 +124,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; - CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); const int32_t currentClusterStatus = MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters); + // TODO: Check about tracks who have clusters in the same row multiple times in different sectors const auto& cluster = clusters[ihit]; + CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); + uint8_t dEdxSubThresholdRow = 255; bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { @@ -141,31 +144,67 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ break; } merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); - FitAddRow(iRow, cluster.sector, iTrk, track, tmpX, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); + if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { + break; + } + FitAddRow(iRow, cluster.sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); } } interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; } lastPropagateRow = cluster.row; - if (currentClusterStatus) { - nMissed++; + int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); + if ((retValProp == -2 && // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha + (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) != 0 || // Cannot rotate to new alpha at all + prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection))) || // propagation fails nonetheless + retValProp) { // failed for other reason but rotation + CADEBUG(printf(" --- break-prop\n")); + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; - if (doInterpolate) { - FitAddRow(cluster.row, cluster.sector, iTrk, track, xx, prop, inFlyDirection, merger, nullptr, false, false, true); - } + NTolerated++; continue; } - - CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + // clang-format off + CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); + CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); + // clang-format on + if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + break; // bad chi2 for the whole track, stop the fit + } if ((uint32_t)interpolationIndex >= interpolation.size) { merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); break; } auto& inter = interpolation.hit[interpolationIndex]; + if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { + prop.InterpolateFill(&inter); + } + + float uncorrectedY = -1e6f; + if (param.rec.tpc.rebuildTrackInFit) { + if (iWay == nWays - 2) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } + if (allowChangeClusters) { + AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit + } + } else if (allowChangeClusters) { + uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); + } else if (param.rec.tpc.rejectEdgeClustersInTrackFit) { + float tmpZ; + merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, mP[0], mP[1], uncorrectedY, tmpZ); + } + + if (currentClusterStatus) { + nMissed++; + nMissed2++; + continue; + } + const bool crossCE = lastUpdateSector != 255 && ((lastUpdateSector < 18) ^ (clusters[ihit].sector < 18)); - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; @@ -227,11 +266,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float tmpX, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) +GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) { - if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { - return; - } if (CAMath::Abs(mP[2]) > GPUCA_MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { return; } @@ -251,46 +287,20 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; const int32_t wayDirection = (iWay & 1) ? -1 : 1; const auto& cluster = clusters[ihit]; - int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - // clang-format off - CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); - // clang-format on - if (retValProp == -2) // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha - { - CADEBUG(printf("REROTATE\n")); - if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) == 0) { - retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - } - } - // clang-format off - CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); - // clang-format on - - float uncorrectedY = -1e6f; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); - } else if (allowChangeClusters) { - uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); - } const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); - const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; - if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - return 1; // bad chi2 for the whole track, stop the fit - } - if (retValProp || sinPhiErr) { + if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; - CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); + CADEBUG(printf(" --- break-sinphi\n")); return 2; // Propagate failed or high incl angle } - CADEBUG(printf("\n")); if (crossCE) { if (param.rec.tpc.addErrorsCECrossing) { @@ -308,6 +318,9 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; + if (param.rec.tpc.rebuildTrackInFit) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); // Will enable rejectChi2 in further rounds + } } else { float err2Y, err2Z; const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; @@ -318,15 +331,15 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - bool rejectChi2 = false; + bool rejectChi2 = (clusterState & GPUTPCGMMergedTrackHit::flagReject); if (param.rec.tpc.mergerInterpolateErrors) { - if (iWay == nWays - 3) { - prop.InterpolateFill(yy, zz, &inter); - } else if (iWay == nWays - 2) { - if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = true; - } else if (!param.rec.tpc.rebuildTrackInFit) { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + if (iWay == nWays - 2) { + if (!param.rec.tpc.rebuildTrackInFit) { + if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + rejectChi2 = true; + } else { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } } } else if (iWay == nWays - 1) { if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { @@ -334,10 +347,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } } else { - rejectChi2 = GetNDF() > 0 && allowChangeClusters; + rejectChi2 = allowChangeClusters; } - if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters + if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedEdge; } else { retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); @@ -368,7 +381,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (retValInt || allowChangeClusters) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); } else if (finalFit) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected ? GPUTPCGMMergedTrackHit::flagRejectDistance : GPUTPCGMMergedTrackHit::flagRejectErr); } if (!retValInt) { nMissed++; @@ -474,7 +487,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre } } } - CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); + CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); } if (nCandidates == 0) { merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; @@ -600,17 +613,17 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; maxDistY2 = (maxDistY2 + mC[0]) * kFactor; maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; - auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { + auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject CADEBUG(, this)](int32_t ih, float y, float z) { float dy = y - projY; float dz = z - projZ; if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { - CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); + CADEBUG(printf("\t\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); if (markReject) { clusters[ih].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; } return false; } else { - CADEBUG(printf("\t\tMerging hit row %d Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ih].row, y, z, dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2))); + CADEBUG(printf("\t\t\tMerging hit row %d Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ih].row, y, z, dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2))); return true; } }; @@ -1062,9 +1075,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() return; } - // clang-format off - CADEBUG(if (DEBUG_SINGLE_TRACK != -1 && iTrk != ((DEBUG_SINGLE_TRACK == -2 && getenv("DEBUG_TRACK")) ? atoi(getenv("DEBUG_TRACK")) : DEBUG_SINGLE_TRACK)) { track.SetNClusters(0); track.SetOK(0); return; } ); - // clang-format on + CADEBUG(if (DEBUG_SINGLE_TRACK != -1 && iTrk != ((DEBUG_SINGLE_TRACK == -2 && getenv("DEBUG_TRACK")) ? atoi(getenv("DEBUG_TRACK")) : DEBUG_SINGLE_TRACK)) { track.SetNClusters(0); track.SetOK(0); return; }); int32_t nTrackHits = track.NClusters(); int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 0985446b64ee0..642047eec5675 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,8 +156,8 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0); - GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float tmpX, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); + GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); From 04036ae98fd6ddb9700675fdc422f91f1dc548c3 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 10:39:27 +0100 Subject: [PATCH 14/25] GPU TPC: Extract CE crossing handling in function, and apply before storing interpolated position --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 50 +++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 3 +- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index a3749c9c893b2..1448edeee026e 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -98,7 +98,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = 0; - uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastUpdateSector = 255; + uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastSector = 255; float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; @@ -134,7 +134,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t dEdxSubThresholdRow = 255; bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastUpdateSector && currentClusterStatus == 0; + bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); if (dodEdx || doAttach || doInterpolate) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; @@ -178,9 +178,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ break; } auto& inter = interpolation.hit[interpolationIndex]; - if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { - prop.InterpolateFill(&inter); - } float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { @@ -197,14 +194,20 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, mP[0], mP[1], uncorrectedY, tmpZ); } + HandleCrossCE(param, cluster.sector, lastSector); + lastSector = cluster.sector; + + if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { + prop.InterpolateFill(&inter); + } + if (currentClusterStatus) { nMissed++; nMissed2++; continue; } - const bool crossCE = lastUpdateSector != 255 && ((lastUpdateSector < 18) ^ (clusters[ihit].sector < 18)); - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; @@ -219,11 +222,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } lastUpdateRow = cluster.row; - lastUpdateSector = cluster.sector; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255) { - StoreLoopPropagation(merger, lastUpdateSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); + StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { @@ -287,7 +289,23 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) +GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() param, const uint8_t sector, const uint8_t& lastSector) +{ + const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (sector < 18)); + if (crossCE) { + if (param.rec.tpc.addErrorsCECrossing) { + if (param.rec.tpc.addErrorsCECrossing >= 2) { + AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); + } else { + AddCovDiagErrors(param.rec.tpc.errorsCECrossing); + } + } else if (mC[2] < 0.5f) { + mC[2] = 0.5f; + } + } +} + +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -302,18 +320,6 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger return 2; // Propagate failed or high incl angle } - if (crossCE) { - if (param.rec.tpc.addErrorsCECrossing) { - if (param.rec.tpc.addErrorsCECrossing >= 2) { - AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); - } else { - AddCovDiagErrors(param.rec.tpc.errorsCECrossing); - } - } else if (mC[2] < 0.5f) { - mC[2] = 0.5f; - } - } - int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 642047eec5675..bfc7919b56b5c 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,8 +156,9 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); + GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); From 09ae94dc6fd092f1a0b2257c05594e409b69c345 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 11:16:51 +0100 Subject: [PATCH 15/25] GPU TPC: If value from other side of interpolation not stored, use only current value --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 3 +- GPU/GPUTracking/Global/GPUChainTracking.cxx | 1 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 36 +++++++++++-------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 9e2e7c79f9980..d083f2e2fa421 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -109,6 +109,8 @@ AddOptionRTC(rejectEdgeClustersMargin, float, 0.f, "", 0, "Margin in cm of Y pos AddOptionRTC(rejectEdgeClustersSigmaMargin, float, 0.f, "", 0, "Margin factor for trackSigmaY when rejecting edge clusters based on uncorrected track Y") AddOptionRTC(trackletMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for tracklet") AddOptionRTC(trackletMinSharedNormFactor, float, 0.f, "", 0, "Max shared defined as trackletMinSharedNormFactor*max(current_nhits,trackletMinSharedNormFactor*minHits,1)") +AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") +AddOptionRTC(rebuildTrackMaxNonIntCov, float, 4.f, "", 0, "Max Err2 allowed for non-interpolated cluster attachment during rebuild") AddOptionRTC(maxTimeBinAboveThresholdIn1000Bin, uint16_t, 500, "", 0, "Except pad from cluster finding if total number of charges in a fragment is above this baseline (disable = 0)") AddOptionRTC(maxConsecTimeBinAboveThreshold, uint16_t, 200, "", 0, "Except pad from cluster finding if number of consecutive charges in a fragment is above this baseline (disable = 0)") AddOptionRTC(noisyPadSaturationThreshold, uint16_t, 700, "", 0, "Threshold where a timebin is considered saturated, disabling the noisy pad check for that pad") @@ -140,7 +142,6 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") -AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 2925bfefc2008..fe2617afe299d 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1016,6 +1016,7 @@ void GPUChainTracking::ApplySyncSettings(GPUSettingsProcessing& proc, GPUSetting { if (syncMode) { rec.useMatLUT = false; + rec.tpc.rebuildTrackMaxNonIntCov = 0.f; } if (proc.rtc.optSpecialCode == -1) { proc.rtc.optSpecialCode = syncMode; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 1448edeee026e..658339fba0cba 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -407,7 +407,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -415,19 +415,24 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre const float stepZ = rowData.HstepZ(); int32_t bin, ny, nz; - float err2Y, err2Z; - param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge - - const float Iz0 = inter.posY - mP[0]; - const float Iz1 = inter.posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; + float ImP0, ImP1, ImC0, ImC2; + if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float Iz0 = inter.posY - mP[0]; + const float Iz1 = inter.posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + ImP0 = mP[0] + Ik00 * Iz0; + ImP1 = mP[1] + Ik11 * Iz1; + ImC0 = mC[0] - Ik00 * mC[0]; + ImC2 = mC[2] - Ik11 * mC[2]; + } else { + ImP0 = mP[0]; + ImP1 = mP[1]; + ImC0 = mC[0]; + ImC2 = mC[2]; + } merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); @@ -436,6 +441,9 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre nCandidates++; } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { + float err2Y, err2Z; + param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + const float kFactor = tracker.GetChiSeedFactor(); const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); From 611a4b272d4f128c43e4687abc73e4ed05f3cee2 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 31 Oct 2025 00:00:45 +0100 Subject: [PATCH 16/25] GPU TPC: Mark clusters of rebuilt track as high incli when interpolated in NoUpdateHighIncl region --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 6 ++- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 5 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 37 +++++++++++++------ 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index aecd98c6a87be..b907693cb14f8 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -2277,7 +2277,11 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads } const auto& best = candidates[candidates[0].best - 1]; const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[best.id - 2]; - outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags)}; + uint8_t flags = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags); + if ((mTrackRebuildHelper[i].highInclRowLow != 255 && j <= mTrackRebuildHelper[i].highInclRowLow) || (mTrackRebuildHelper[i].highInclRowHigh != 255 && j >= mTrackRebuildHelper[i].highInclRowHigh)) { + flags |= GPUTPCGMMergedTrackHit::flagHighIncl; + } + outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = flags}; written++; CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, best.id - 2)); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 0d2b190203450..50b1aec8dddd4 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -87,7 +87,9 @@ class GPUTPCGMMerger : public GPUProcessor }; struct trackRebuildHelper { - bool reverse; + uint8_t reverse; + uint8_t highInclRowLow; + uint8_t highInclRowHigh; }; struct tmpSort { @@ -123,6 +125,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return mClusters; } GPUhdi() trackCluster* ClusterCandidates() { return mClusterCandidates; } + GPUhdi() trackRebuildHelper* TrackRebuildHelper() { return mTrackRebuildHelper; } GPUhdi() int32_t* HitWeights() { return mHitWeights; } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 658339fba0cba..1e98bd8777414 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -65,6 +65,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing size interpolation.hit[i].errorY = -1; } + if (param.rec.tpc.rebuildTrackInFit) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = 255; + merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; + } } const int32_t nWays = param.rec.tpc.nWays; @@ -207,6 +211,28 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } + const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); + if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + nMissed2++; + CADEBUG(printf(" --- break-sinphi\n")); + NTolerated++; + const bool inward = clusters[0].row > clusters[maxN - 1].row; + const bool markHighIncl = (mP[2] > 0) ^ (mP[4] < 0) ^ inward ^ (iWay & 1); + if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { + if (inward ^ (iWay & 1)) { + if (merger.TrackRebuildHelper()[iTrk].highInclRowLow == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = cluster.row; + } + } else { + if (merger.TrackRebuildHelper()[iTrk].highInclRowHigh == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; + } + } + } + continue; + } + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); @@ -216,9 +242,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ covYYUpd = mC[0]; } else if (retValHit == 1) { break; - } else if (retValHit == 2) { - NTolerated++; - continue; } lastUpdateRow = cluster.row; @@ -312,14 +335,6 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger const int32_t wayDirection = (iWay & 1) ? -1 : 1; const auto& cluster = clusters[ihit]; - const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); - if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; - CADEBUG(printf(" --- break-sinphi\n")); - return 2; // Propagate failed or high incl angle - } - int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { From 900d462b275c2bb573d8221ce8a757f4ea2ea6b6 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 18:39:10 +0100 Subject: [PATCH 17/25] GPU TPC: Pick up best hit from interpolation only, if current position not available --- GPU/GPUTracking/Global/GPUChainTracking.cxx | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 54 +++++++++++++++---- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 3 +- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index fe2617afe299d..cff767de7f484 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1016,7 +1016,7 @@ void GPUChainTracking::ApplySyncSettings(GPUSettingsProcessing& proc, GPUSetting { if (syncMode) { rec.useMatLUT = false; - rec.tpc.rebuildTrackMaxNonIntCov = 0.f; + rec.tpc.rebuildTrackMaxNonIntCov = 0.f; // TODO: Check if this yields a performance benefit } if (proc.rtc.optSpecialCode == -1) { proc.rtc.optSpecialCode = syncMode; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 1e98bd8777414..de3b293bdcd9a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -107,7 +107,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart - wayDirection; ihit >= 0 && ihit < maxN; ihit += wayDirection) { + for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart; ihit >= 0 && ihit < maxN; ihit += wayDirection, interpolationIndex += wayDirection) { if (!param.rec.tpc.rebuildTrackInFit || rebuilt) { if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); @@ -124,7 +124,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool allowChangeClusters = finalOutInFit && (nWays == 1 || ((iWay & 1) ? (ihit <= CAMath::Max(maxN / 2, maxN - 30)) : (ihit >= CAMath::Min(maxN / 2, 30)))); int32_t ihitMergeFirst = ihit; - interpolationIndex += wayDirection; uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; @@ -158,6 +157,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } lastPropagateRow = cluster.row; + auto& inter = interpolation.hit[interpolationIndex]; int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); if ((retValProp == -2 && // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) != 0 || // Cannot rotate to new alpha at all @@ -167,6 +167,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; NTolerated++; + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); // TODO: mark clusters from single-sided finding as dubious, and apply stricter restriction cut later + } continue; } // clang-format off @@ -174,6 +177,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } break; // bad chi2 for the whole track, stop the fit } @@ -181,12 +187,11 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); break; } - auto& inter = interpolation.hit[interpolationIndex]; float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { if (iWay == nWays - 2) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); } if (allowChangeClusters) { AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit @@ -241,6 +246,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ N++; covYYUpd = mC[0]; } else if (retValHit == 1) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } break; } @@ -414,7 +422,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } -GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly) { const GPUParam& GPUrestrict() param = merger.Param(); const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); @@ -422,7 +430,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { + if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -431,9 +439,14 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre int32_t bin, ny, nz; float ImP0, ImP1, ImC0, ImC2; - if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - const float Iz0 = inter.posY - mP[0]; - const float Iz1 = inter.posZ + deltaZ - mP[1]; + if (interOnly) { + ImP0 = (float)inter.posY; + ImP1 = (float)inter.posZ + deltaZ; + ImC0 = (float)inter.errorY; + ImC2 = (float)inter.errorZ; + } else if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float Iz0 = (float)inter.posY - mP[0]; + const float Iz1 = (float)inter.posZ + deltaZ - mP[1]; const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); const float Ik00 = mC[0] * Iw0; @@ -518,14 +531,33 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre } CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); } - if (nCandidates == 0) { + if (nCandidates == 0 && !interOnly) { // TODO: remove the interOnly here when rebuilding is fully working merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; } } return uncorrectedY; } -GPUdii() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) +GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrors& GPUrestrict() interpolation, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +{ + for (; ihit >= 0; ihit--, interpolationIndex--) { + while (ihit > 0 && clusters[ihit].row == clusters[ihit - 1].row && clusters[ihit].sector == clusters[ihit - 1].sector) { + ihit--; + } + const auto& cluster = clusters[ihit]; + if (CAMath::Abs(cluster.row - lastRow) > 1) { + interpolationIndex -= CAMath::Abs(cluster.row - lastRow) - 1; + } + lastRow = cluster.row; + if (interpolationIndex < 0) { // TODO: Is this check needed? + return; + } + auto inter = interpolation.hit[interpolationIndex]; + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); + } +} + +GPUd() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) { const GPUParam& GPUrestrict() param = merger.Param(); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index bfc7919b56b5c..cbec62fac5751 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -163,7 +163,8 @@ class GPUTPCGMTrackParam GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); - GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); + GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly); + GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); GPUd() float AttachClusters(const GPUTPCGMMerger& merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use GPUd() float AttachClusters(const GPUTPCGMMerger& merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); From eec164981ec2d8f61cbc2af8857f4c083d678532 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 31 Oct 2025 13:04:02 +0100 Subject: [PATCH 18/25] GPU TPC: Interpolate missing pad rows when rebuilding track --- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 4 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 50 ++++++++++++------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index c34e5384e3cef..ebab5fc5320f9 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -34,11 +34,13 @@ enum attachTypes { attachProtect = 0x80000000, struct InterpolationErrorHit { float posY, posZ; GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A errorY, errorZ; + GPUdi() bool isValid() const { return errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0; } + GPUdi() void markInvalid() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-1; } }; struct InterpolationErrors { - InterpolationErrorHit hit[GPUCA_MERGER_MAX_TRACK_CLUSTERS]; static constexpr size_t size = GPUCA_MERGER_MAX_TRACK_CLUSTERS; + InterpolationErrorHit hit[size]; }; struct GPUResolveSharedMemory : public GPUKernelTemplate::GPUSharedMemoryScan64 { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index de3b293bdcd9a..cb794675ff7a9 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -62,12 +62,11 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { - for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing size - interpolation.hit[i].errorY = -1; + for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing range + interpolation.hit[i].markInvalid(); } if (param.rec.tpc.rebuildTrackInFit) { - merger.TrackRebuildHelper()[iTrk].highInclRowLow = 255; - merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; + merger.TrackRebuildHelper()[iTrk].highInclRowLow = merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; } } @@ -135,22 +134,24 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); uint8_t dEdxSubThresholdRow = 255; - bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; - bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); - if (dodEdx || doAttach || doInterpolate) { + const bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; + const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); + const uint8_t doInterpolate = (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0); + if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; - for (int32_t iRow = lastPropagateRow + step; iRow != cluster.row; iRow += step) { + uint8_t sector = lastSector; + + for (int32_t iRow = lastPropagateRow + step, index = interpolationIndex; iRow != cluster.row; iRow += step, index += wayDirection) { float tmpX, tmpY, tmpZ; - if (prop.GetPropagatedYZ(mX - GPUTPCGeometry::Row2X(iRow - step) + GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { break; } - merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); + merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoX(sector, iRow, tmpY, tmpZ, tmpX); if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { break; } - FitAddRow(iRow, cluster.sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); + FitAddRow(iRow, sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate, interpolation.hit[index], deltaZ, sumInvSqrtCharge, nAvgCharge); } } interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; @@ -234,6 +235,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; } } + // TODO: We can perhaps break here, if we pick up remaining rows } continue; } @@ -299,7 +301,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) +GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge) { if (CAMath::Abs(mP[2]) > GPUCA_MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { return; @@ -318,6 +320,16 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se if (doAttach) { AttachClusters(merger, sector, iRow, iTrk, track.Leg() == 0, prop); } + if (doInterpolate) { + if (doInterpolate == 2) { + if (inter.isValid() && CAMath::Abs(inter.posY - mP[0]) < 4.f) { + inter.markInvalid(); + } + FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); + } else { + prop.InterpolateFill(&inter); + } + } } GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() param, const uint8_t sector, const uint8_t& lastSector) @@ -364,10 +376,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (param.rec.tpc.mergerInterpolateErrors) { if (iWay == nWays - 2) { if (!param.rec.tpc.rebuildTrackInFit) { - if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = true; - } else { + if (inter.isValid()) { retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } else { + rejectChi2 = true; } } } else if (iWay == nWays - 1) { @@ -430,7 +442,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { + if (rowData.NHits() && (inter.isValid() || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransformHelper->getCorrMap()->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -444,7 +456,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre ImP1 = (float)inter.posZ + deltaZ; ImC0 = (float)inter.errorY; ImC2 = (float)inter.errorZ; - } else if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + } else if (inter.isValid()) { const float Iz0 = (float)inter.posY - mP[0]; const float Iz1 = (float)inter.posZ + deltaZ - mP[1]; const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); @@ -521,7 +533,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; - } + } // TODO: Downweight non-interpolated candidates merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = row, .sector = sector, .error = err, .weight = 0, .best = 0}; nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index cbec62fac5751..eea972fa14c51 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -157,7 +157,7 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); - GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); + GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); From a68a13604fec23d3b2ba94aaae6f054dd7c4858d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 31 Oct 2025 14:07:16 +0100 Subject: [PATCH 19/25] Revert "GPU TPC: Remove option to retry refit if cluster rejection breaks the track, we will anyway rebuild the track in the future" This reverts commit 5c3d113f80687ecfcb3bf106b0fc7ad274fceda9. We can actually properly retry also for rebuilt tracks, we just run the retry on the last iteration after rebuilding. --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + .../Global/GPUChainTrackingMerger.cxx | 4 ++ GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 5 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 3 + GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 7 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 62 ++++++++++++------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 6 +- GPU/GPUTracking/qa/GPUQA.cxx | 2 +- 8 files changed, 60 insertions(+), 30 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index d083f2e2fa421..331d1589f31ff 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -160,6 +160,7 @@ AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation inst AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF") AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF of track for non-interpolated reject (both for chi2 and absolute distance)") AddOptionRTC(mergeCE, uint8_t, 1, "", 0, "Merge tracks accross the central electrode") +AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without cluster rejection when fit fails") AddOptionRTC(enablePID, int8_t, 1, "", 0, "Enable PID response") AddOptionRTC(PID_useNsigma, int8_t, 1, "", 0, "Use nSigma instead of absolute distance in PID response") AddOptionRTC(adddEdxSubThresholdClusters, int8_t, 1, "", 0, "Add sub threshold clusters in TPC dEdx computation") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index bf6d659adf86f..89a766c094df1 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -240,6 +240,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRebuiltTracks, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); } + if (param().rec.tpc.retryRefit) { + runKernel(GetGridAuto(0), -1, param().rec.tpc.rebuildTrackInFit); + } + runKernel(GetGridAuto(0)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRefit, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index b907693cb14f8..cc45a4a4856ed 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -402,13 +402,16 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mTrackSort, mNMaxTracks); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway + computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks does not save memory, other parts are larger anyway if (mRec->GetParam().rec.tpc.rebuildTrackInFit) { computePointerWithAlignment(mem, mSharedCount2, mNClusters); computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUCA_ROW_COUNT * mRec->GetParam().rec.tpc.rebuildTrackInFitClusterCandidates); computePointerWithAlignment(mem, mTrackRebuildHelper, mNMaxTracks); computePointerWithAlignment(mem, mHitWeights, mNClusters); } + if (mRec->GetParam().rec.tpc.retryRefit) { + computePointerWithAlignment(mem, mRetryRefitIds, mNMaxTracks); + } memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 50b1aec8dddd4..6ee164435c81e 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -66,6 +66,7 @@ class GPUTPCGMMerger : public GPUProcessor static constexpr const int32_t NSECTORS = GPUCA_NSECTORS; //* N sectors struct memory { + GPUAtomic(uint32_t) nRetryRefit; GPUAtomic(uint32_t) nLoopData; GPUAtomic(uint32_t) nUnpackedTracks; GPUAtomic(uint32_t) nMergedTracks; @@ -130,6 +131,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } + GPUhdi() uint32_t* RetryRefitIds() const { return mRetryRefitIds; } GPUhdi() uint8_t* ClusterStateExt() const { return mClusterStateExt; } GPUhdi() GPUTPCGMLoopData* LoopData() const { return mLoopData; } GPUhdi() memory* Memory() const { return mMemory; } @@ -312,6 +314,7 @@ class GPUTPCGMMerger : public GPUProcessor gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRangeMemory = nullptr; // memory for border tracks gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRange[NSECTORS]; // memory for border tracks memory* mMemory = nullptr; + uint32_t* mRetryRefitIds = nullptr; GPUTPCGMLoopData* mLoopData = nullptr; }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 12b50f9f4cee2..417068de7064b 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -21,9 +21,10 @@ using namespace o2::gpu; template <> GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode, int32_t rebuilt) { - GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { - const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt); + const int32_t iEnd = mode == -1 ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); + GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, iEnd, { + const int32_t i = mode == -1 ? merger.RetryRefitIds()[ii] : (mode ? merger.TrackOrderProcess()[ii] : ii); + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt, mode == -1); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index cb794675ff7a9..7d42ddd370d3a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -48,7 +48,7 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt, bool retryAttempt) { static constexpr float maxSinPhi = GPUCA_MAX_SIN_PHI; @@ -92,7 +92,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool finalFit = iWay == nWays - 1; ResetCovariance(); - prop.SetSeedingErrors(!(refit)); + prop.SetSeedingErrors(!(refit && !retryAttempt)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); @@ -240,7 +240,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY, retryAttempt); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; @@ -257,7 +257,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ lastUpdateRow = cluster.row; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255) { + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -348,7 +348,7 @@ GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() pa } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY, bool retryAttempt) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -372,23 +372,25 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - bool rejectChi2 = (clusterState & GPUTPCGMMergedTrackHit::flagReject); - if (param.rec.tpc.mergerInterpolateErrors) { - if (iWay == nWays - 2) { - if (!param.rec.tpc.rebuildTrackInFit) { - if (inter.isValid()) { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); - } else { + bool rejectChi2 = !retryAttempt && (clusterState & GPUTPCGMMergedTrackHit::flagReject); + if (!retryAttempt) { + if (param.rec.tpc.mergerInterpolateErrors) { + if (iWay == nWays - 2) { + if (!param.rec.tpc.rebuildTrackInFit) { + if (inter.isValid()) { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } else { + rejectChi2 = true; + } + } + } else if (iWay == nWays - 1) { + if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { rejectChi2 = true; } } - } else if (iWay == nWays - 1) { - if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { - rejectChi2 = true; - } + } else { + rejectChi2 = allowChangeClusters; } - } else { - rejectChi2 = allowChangeClusters; } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { @@ -894,6 +896,10 @@ GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger& GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t loopIdx) { + GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; + if (!merger.MergedTracks()[data.track].OK()) { + return; + } GPUTPCGMPropagator prop; prop.SetMaterialTPC(); prop.SetPolynomialField(&merger.Param().polynomialField); @@ -903,7 +909,6 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr prop.SetFitInProjections(true); prop.SetPropagateBzOnly(false); - GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; prop.SetTrack(&data.param, data.alpha); if (merger.Param().rec.tpc.looperFollowMode == 1) { data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.row, data.track, data.outwards); @@ -1142,7 +1147,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt, bool retryAttempt) { if (!track.OK()) { return; @@ -1154,16 +1159,29 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt); + bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt, retryAttempt); CADEBUG(if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); + if (!ok && (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) && !retryAttempt && merger.Param().rec.tpc.retryRefit) { + for (uint32_t i = 0; i < track.NClusters(); i++) { + merger.Clusters()[track.FirstClusterRef() + i].state &= GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + } + CADEBUG(printf("Track rejected, marking for retry\n")); + uint32_t nRefit = CAMath::AtomicAdd(&merger.Memory()->nRetryRefit, 1u); + merger.RetryRefitIds()[nRefit] = iTrk; + return; + } + if (retryAttempt && (t.mNDF < 10 || t.mChi2 / t.mNDF >= 6.f)) { + ok = false; + } + if (CAMath::Abs(t.QPt()) < 1.e-4f) { t.QPt() = CAMath::Copysign(1.e-4f, t.QPt()); } CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger.MergedTracks()[iTrk].Looper()); }); - track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? + track.SetOK(ok); if (t.GetNDF() <= 0 && !rebuilt && merger.Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? track.Param().NDF() = 0; } else { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index eea972fa14c51..a43542c813493 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -154,12 +154,12 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt, bool retryAttempt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt, bool retryAttempt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 6f29fa7d4efb3..733198ec6358b 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -2906,7 +2906,7 @@ int32_t GPUQA::DrawQAHistograms(TObjArray* qcout) } } - if (mQATasks & taskTrackStatistics) { + if (mQATasks & taskTrackStatistics) { // TODO: Add plot for Chi2/NDF, and NCl correct and fake MC histograms // Process track statistic histograms float tmpMax = 0.; for (int32_t k = 0; k < ConfigNumInputs; k++) { // TODO: Simplify this drawing, avoid copy&paste From e299926ac6f21fa02756797ed5c1267c3959949b Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 11:06:13 +0100 Subject: [PATCH 20/25] GPU TPC: Relax chi2cut for single-pad clusters --- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index be1dbabee2ae4..301c50c3c83d0 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -108,13 +108,19 @@ class GPUTPCGMPropagator GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; GPUd() static int32_t RejectCluster(float chiY, float chiZ, uint8_t clusterState) { - if (chiY > 9.f || chiZ > 9.f) { // TODO: Check how a track can have chi2/ncl > 18 + if (chiY > 9.f || chiZ > 9.f) { return 2; } - if ((chiY > 6.25f || chiZ > 6.25f) && (clusterState & (GPUTPCGMMergedTrackHit::flagSplit | GPUTPCGMMergedTrackHit::flagShared))) { + if (chiZ > 6.25f && (clusterState & (GPUTPCGMMergedTrackHit::flagSplit | GPUTPCGMMergedTrackHit::flagShared | GPUTPCGMMergedTrackHit::flagEdge | GPUTPCGMMergedTrackHit::flagSingle))) { return 2; } - if ((chiY > 1.f || chiZ > 6.25f) && (clusterState & (GPUTPCGMMergedTrackHit::flagEdge | GPUTPCGMMergedTrackHit::flagSingle))) { + if (chiY > 6.25f && (clusterState & (GPUTPCGMMergedTrackHit::flagSplit | GPUTPCGMMergedTrackHit::flagShared))) { + return 2; + } + if (chiY > 1.f && (clusterState & GPUTPCGMMergedTrackHit::flagEdge)) { + return 2; + } + if (chiY > 2.f && (clusterState & GPUTPCGMMergedTrackHit::flagSingle)) { return 2; } return 0; From e2fac1cbb43205439ba338d7873e58954485dc4d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 15:41:14 +0100 Subject: [PATCH 21/25] GPU TPC: Clean up and fix resetT0 code --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 19 ++++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 13 +------------ 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 7d42ddd370d3a..23abe77da347d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -86,7 +86,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ StoreOuter(&track.OuterParam(), prop.GetAlpha()); } - int32_t resetT0 = initResetT0(); + int32_t resetT0 = CAMath::Max(10, CAMath::Min(40, 150.f / CAMath::Abs(mP[4]))); + ; const bool refit = (nWays == 1 || iWay >= 1); const bool finalOutInFit = iWay + 2 >= nWays; const bool finalFit = iWay == nWays - 1; @@ -240,9 +241,15 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY, retryAttempt); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, uncorrectedY, retryAttempt); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); + if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && CAMath::Square(mP[0] - prop.Model().Y()) + CAMath::Square(mP[1] - prop.Model().Z()) > 1) { + CADEBUG(printf("Reinit linearization\n")); + prop.SetTrack(this, prop.GetAlpha()); + resetT0 = 20; + } + ihitStart = ihit; interpolatedStart = interpolationIndex; N++; @@ -348,7 +355,7 @@ GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() pa } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY, bool retryAttempt) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, float uncorrectedY, bool retryAttempt) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -413,12 +420,6 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger lastUpdateX = mX; nMissed = nMissed2 = 0; UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - float dy = mP[0] - prop.Model().Y(); - float dz = mP[1] - prop.Model().Z(); - if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { - CADEBUG(printf("Reinit linearization\n")); - prop.SetTrack(this, prop.GetAlpha()); - } return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track if (retValInt || allowChangeClusters) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index a43542c813493..d0f3e57b5faae 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,7 +156,7 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt, bool retryAttempt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY, bool retryAttempt); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt, bool retryAttempt); @@ -229,8 +229,6 @@ class GPUTPCGMTrackParam } private: - GPUd() int32_t initResetT0(); - float mX; // x position float mTOffset; // Z offset with early transform, T offset otherwise float mP[5]; // 'active' track parameters: Y, Z, SinPhi, DzDs, q/Pt @@ -248,15 +246,6 @@ struct GPUTPCGMLoopData { uint8_t outwards; }; -GPUdi() int32_t GPUTPCGMTrackParam::initResetT0() -{ - const float absQPt = CAMath::Abs(mP[4]); - if (absQPt < (150.f / 40.f)) { - return 150.f / 40.f; - } - return CAMath::Max(10.f, 150.f / mP[4]); -} - GPUdi() void GPUTPCGMTrackParam::ResetCovariance() { mC[0] = 100.f; From a2b6e9a45eaf0767b7a9be8ba4d2ad7c88965b19 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 15:42:27 +0100 Subject: [PATCH 22/25] GPU TPC: Don't break for high sinPhi if only the linearization model is out of range, but instead reinitialize the model --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 23abe77da347d..68422e3336a91 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -219,26 +219,31 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); - if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; - CADEBUG(printf(" --- break-sinphi\n")); - NTolerated++; + if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { // TODO: If NDF is large enough, and mP[2] is not yet out of range, reinit linearization const bool inward = clusters[0].row > clusters[maxN - 1].row; const bool markHighIncl = (mP[2] > 0) ^ (mP[4] < 0) ^ inward ^ (iWay & 1); - if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { - if (inward ^ (iWay & 1)) { - if (merger.TrackRebuildHelper()[iTrk].highInclRowLow == 255) { - merger.TrackRebuildHelper()[iTrk].highInclRowLow = cluster.row; - } - } else { - if (merger.TrackRebuildHelper()[iTrk].highInclRowHigh == 255) { - merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; + if (mNDF > 10 && CAMath::Abs(mP[2]) < maxSinForUpdate && markHighIncl) { + CADEBUG(printf("Reinit linearization\n")); + prop.SetTrack(this, prop.GetAlpha()); + } else { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + nMissed2++; + CADEBUG(printf(" --- break-sinphi\n")); + NTolerated++; + if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { + if (inward ^ (iWay & 1)) { + if (merger.TrackRebuildHelper()[iTrk].highInclRowLow == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = cluster.row; + } + } else { + if (merger.TrackRebuildHelper()[iTrk].highInclRowHigh == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; + } } + // TODO: We can perhaps break here, if we pick up remaining rows } - // TODO: We can perhaps break here, if we pick up remaining rows + continue; } - continue; } int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, uncorrectedY, retryAttempt); From c2dc568908e4d327abf372df96e7a8becda888f6 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 18 Nov 2025 15:37:23 +0100 Subject: [PATCH 23/25] GPU: Add option to steer which attachment steps to run during rebuild --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 7 +++--- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 22 ++++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 21 +++++++++--------- .../Standalone/Benchmark/standalone.cxx | 2 +- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 331d1589f31ff..b5ff4c8f6248f 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -142,13 +142,14 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") +AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit)") +AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") -AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following)") -AddOptionRTC(looperFollowMode, uint8_t, 1, "", 0, "0 = simple, 1 = advances, for disabling use disableRefitAttachment = 4") +AddOptionRTC(looperFollowMode, uint8_t, 1, "", 0, "0 = simple, 1 = advances, for disabling use disableMarkAdjacent = 4") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression (0 = no, 1 = strategy A, 2 = strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits (1=truncate charge/width LSB, 2=differences, 4=track-model)") @@ -171,7 +172,7 @@ AddOptionRTC(rejectEdgeClustersInTrackFit, int8_t, 0, "", 0, "Reject edge cluste AddOptionRTC(tubeExtraProtectMinRow, uint8_t, 20, "", 0, "Increase Protection, decrease removal by factor 2, when below this row") AddOptionRTC(tubeExtraProtectEdgePads, uint8_t, 2, "", 0, "Increase Protection, decrease removal by factor 2, when on this number of pads from the edge") -AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet hand AddOptionArrayRTC +AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index cc45a4a4856ed..451ad53bc79d9 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -2155,16 +2155,18 @@ GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, continue; } mTrackRebuildHelper[i].reverse = mClusters[trk.FirstClusterRef()].row < mClusters[trk.FirstClusterRef() + trk.NClusters() - 1].row; - int lastRow = -1; - for (uint32_t j = 0; j < trk.NClusters(); j++) { - const auto& cl = mClusters[trk.FirstClusterRef() + j]; - auto* candidates = &mClusterCandidates[(i * GPUCA_ROW_COUNT + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { - candidates[0].id = cl.num + 2; - candidates[0].best = 128; - candidates[0].weight = cl.state; - candidates[0].sector = cl.sector; - lastRow = cl.row; + if (!(Param().rec.tpc.disableRebuildAttachment & 8)) { + int lastRow = -1; + for (uint32_t j = 0; j < trk.NClusters(); j++) { + const auto& cl = mClusters[trk.FirstClusterRef() + j]; + auto* candidates = &mClusterCandidates[(i * GPUCA_ROW_COUNT + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { + candidates[0].id = cl.num + 2; + candidates[0].best = 128; + candidates[0].weight = cl.state; + candidates[0].sector = cl.sector; + lastRow = cl.row; + } } } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 68422e3336a91..e9f19253f8d39 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -137,8 +137,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t dEdxSubThresholdRow = 255; if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { const bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; - const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); - const uint8_t doInterpolate = (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0); + const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableMarkAdjacent & 2); + const uint8_t doInterpolate = (param.rec.tpc.disableRebuildAttachment & 2) ? 0 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0)); if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; uint8_t sector = lastSector; @@ -169,7 +169,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; NTolerated++; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); // TODO: mark clusters from single-sided finding as dubious, and apply stricter restriction cut later } continue; @@ -179,7 +179,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } break; // bad chi2 for the whole track, stop the fit @@ -192,15 +192,16 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { - if (iWay == nWays - 2) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); + if (iWay == nWays - 2 && !(param.rec.tpc.disableRebuildAttachment & 1)) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); // TODO: Check whether we do not need to use track Y for some rejection steps } if (allowChangeClusters) { AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit } } else if (allowChangeClusters) { uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); - } else if (param.rec.tpc.rejectEdgeClustersInTrackFit) { + } + if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY <= -1e6f) { float tmpZ; merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, mP[0], mP[1], uncorrectedY, tmpZ); } @@ -260,7 +261,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ N++; covYYUpd = mC[0]; } else if (retValHit == 1) { - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } break; @@ -269,7 +270,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ lastUpdateRow = cluster.row; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255 && !retryAttempt) { + if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -773,7 +774,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) { const GPUParam& GPUrestrict() param = merger.Param(); - if (param.rec.tpc.disableRefitAttachment & 1) { + if (param.rec.tpc.disableMarkAdjacent & 1) { return -1e6f; } const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); diff --git a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx index 78bc41ec7350a..6cbb9b75284e6 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx +++ b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx @@ -462,7 +462,7 @@ int32_t SetupReconstruction() if (recSet.tpc.rejectionStrategy >= GPUSettings::RejectionStrategyB) { procSet.tpcInputWithClusterRejection = 1; } - recSet.tpc.disableRefitAttachment = 0xFF; + recSet.tpc.disableMarkAdjacent = 0xFF; recSet.maxTrackQPtB5 = CAMath::Min(recSet.maxTrackQPtB5, recSet.tpc.rejectQPtB5); GPUChainTracking::ApplySyncSettings(procSet, recSet, steps.steps, false, configStandalone.rundEdx); recAsync->SetSettings(&grp, &recSet, &procSet, &steps); From e4416a0f6b850ba3972aa02f8fef7969b14ffbdf Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 19:19:38 +0100 Subject: [PATCH 24/25] GPU TPC: Extrapolate track inward and outward when rebuilding --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 6 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 114 +++++++++++++++++- 2 files changed, 113 insertions(+), 7 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index b5ff4c8f6248f..04e452fa59e42 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -142,7 +142,7 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") -AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit)") +AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit, 16: extrapolation)") AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") @@ -171,7 +171,9 @@ AddOptionRTC(rejectEdgeClustersInSeeding, int8_t, 0, "", 0, "Reject edge cluster AddOptionRTC(rejectEdgeClustersInTrackFit, int8_t, 0, "", 0, "Reject edge clusters based on uncorrected track Y during track fit") AddOptionRTC(tubeExtraProtectMinRow, uint8_t, 20, "", 0, "Increase Protection, decrease removal by factor 2, when below this row") AddOptionRTC(tubeExtraProtectEdgePads, uint8_t, 2, "", 0, "Increase Protection, decrease removal by factor 2, when on this number of pads from the edge") - +AddOptionRTC(rebuildTrackExtrMaxMissingRows, uint8_t, 15, "", 0, "Maximum total number of rows allowed to be missing during track extrapolation") +AddOptionRTC(rebuildTrackExtrMinConsecGoodRows, uint8_t, 6, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") +AddOptionRTC(rebuildTrackExtrConsecGoodRowsMaxGap, uint8_t, 1, "", 0, "Max row gap size to consider the segment as consecutive rows") AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index e9f19253f8d39..12cfcf9d6e330 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -51,6 +51,7 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt, bool retryAttempt) { static constexpr float maxSinPhi = GPUCA_MAX_SIN_PHI; + const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); const GPUParam& GPUrestrict() param = merger.Param(); GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -219,7 +220,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { // TODO: If NDF is large enough, and mP[2] is not yet out of range, reinit linearization const bool inward = clusters[0].row > clusters[maxN - 1].row; const bool markHighIncl = (mP[2] > 0) ^ (mP[4] < 0) ^ inward ^ (iWay & 1); @@ -270,16 +270,120 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ lastUpdateRow = cluster.row; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { - StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); - CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); - } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { deltaZ = ShiftZ(clusters, merger, maxN); } else { deltaZ = 0.f; } + if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && iWay >= nWays - 3 && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { + const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; + int32_t sector = lastSector; + uint8_t rowGapActive = 0, rowGapTotal = 0, missingRowsTotal = 0; + uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow; + uint8_t consecGoodRows = param.rec.tpc.rebuildTrackExtrMinConsecGoodRows, consecGoodRowsMissing = 0; + prop.SetTrack(this, prop.GetAlpha()); + for (int32_t iRow = lastPropagateRow + up; iRow >= 0 && iRow < GPUCA_ROW_COUNT; iRow += up) { // TODO: Try to reduce some variables to int8/uint8 to save registers + bool fail = false; + for (int32_t iAttempt = 0; iAttempt < 2; iAttempt++) { + float tmpX, tmpY, tmpZ; + if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + fail = true; + break; + } + merger.GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoX(sector, iRow, tmpY, tmpZ, tmpX); + if (prop.PropagateToXAlpha(tmpX, param.Alpha(sector), inFlyDirection)) { + fail = true; + break; + } + if (CAMath::Abs(mP[2]) > maxSinForUpdate) { + fail = true; + break; + } + if (CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f) + 0.1f) { + if (iAttempt) { + fail = true; + break; + } + const int32_t sectorSide = sector >= (GPUCA_NSECTORS / 2) ? (GPUCA_NSECTORS / 2) : 0; + if (mP[0] > 0) { + if (++sector >= sectorSide + 18) { + sector -= 18; + } + } else { + if (--sector < sectorSide) { + sector += 18; + } + } + } + } + if (fail) { + break; + } + CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Propaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", sector, iRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + gputpcgmmergertypes::InterpolationErrorHit inter; + inter.markInvalid(); + float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + if (candidate.id >= 2) { + lastExtrapolateRow = iRow; + float err2Y, err2Z, xx, yy, zz; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[candidate.id - 2]; + merger.GetConstantMem()->calibObjects.fastTransformHelper->Transform(sector, iRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection)) { + candidate.best = -1; + break; + } + const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? cl.getTime() : -1.f; // TODO: When is it possible that we do not have clusterNative? + const float invSqrtCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(cl.qMax) : 0.f; + const float invCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? (1.f / cl.qMax) : 0.f; + float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; + invAvgCharge *= invAvgCharge; + const uint8_t clusterState = cl.getFlags(); + prop.GetErr2(err2Y, err2Z, param, zz, iRow, clusterState, sector, time, invAvgCharge, invCharge); + CADEBUG(printf("\t%21sResiduals %8.3f %8.3f --- Errors %8.3f %8.3f\n", "", yy - mP[0], zz - mP[1], CAMath::Sqrt(err2Y), CAMath::Sqrt(err2Z))); + uint32_t retValUpd = prop.Update(yy, zz, iRow, param, clusterState, true, refit, err2Y, err2Z); + CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); + if (retValUpd == 1) { + candidate.best = -1; + break; + } + if (++consecGoodRows >= param.rec.tpc.rebuildTrackExtrMinConsecGoodRows) { + lastGoodRow = iRow; + consecGoodRowsMissing = 0; + } + rowGapActive = rowGapTotal = 0; + } else { + if (++missingRowsTotal > param.rec.tpc.rebuildTrackExtrMaxMissingRows) { + break; + } + if (++rowGapTotal > param.rec.tpc.trackFollowingMaxRowGap) { + consecGoodRows = consecGoodRowsMissing = 0; + } + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, uncorrectedY)); + if (pad < GPUTPCGeometry::NPads(iRow) && (!merger.GetConstantMem()->calibObjects.dEdxCalibContainer || !merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { + if (rowGapActive++ >= param.rec.tpc.trackFollowingMaxRowGap) { + break; + } + if (consecGoodRowsMissing++ >= param.rec.tpc.rebuildTrackExtrConsecGoodRowsMaxGap) { + consecGoodRows = consecGoodRowsMissing = 0; + } + } + } + } + if (lastGoodRow != lastExtrapolateRow) { + for (int32_t iRow = lastGoodRow + up; iRow != lastExtrapolateRow + up; iRow += up) { + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + candidate.best = -1; + } + } + } + + if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { + StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); + CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); + } + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { Alpha = prop.GetAlpha(); if (ihitStart != 0) { From 2b03a37d94fcc2a9c1244df7f8b818a73bd8dccf Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 4 Dec 2025 12:05:46 +0100 Subject: [PATCH 25/25] GPU: Add debug option to create temporary MC labels for collected merged tracks --- GPU/GPUTracking/Base/GPUReconstructionCPU.cxx | 4 +-- GPU/GPUTracking/Definitions/GPUSettingsList.h | 14 ++++++-- .../Global/GPUChainTrackingMerger.cxx | 3 ++ GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 34 ++++++++++++++++++- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 3 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 13 +++++++ GPU/GPUTracking/qa/GPUQAHelper.h | 11 ++++++ 7 files changed, 76 insertions(+), 6 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx index 409c28b8bf328..6a3378e059b25 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx +++ b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx @@ -215,10 +215,10 @@ int32_t GPUReconstructionCPU::ExitDevice() int32_t GPUReconstructionCPU::RunChains() { mMemoryScalers->temporaryFactor = 1.; - if (GetProcessingSettings().memoryScalingFuzz) { + if (GetProcessingSettings().debug.memoryScalingFuzz) { static std::mt19937 rng; static std::uniform_int_distribution dist(0, 1000000); - uint64_t fuzzFactor = GetProcessingSettings().memoryScalingFuzz == 1 ? dist(rng) : GetProcessingSettings().memoryScalingFuzz; + uint64_t fuzzFactor = GetProcessingSettings().debug.memoryScalingFuzz == 1 ? dist(rng) : GetProcessingSettings().debug.memoryScalingFuzz; GPUInfo("Fuzzing memory scaling factor with %lu", fuzzFactor); mMemoryScalers->fuzzScalingFactor(fuzzFactor); } diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 04e452fa59e42..14a8c895331fb 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -39,7 +39,7 @@ BeginNamespace(gpu) // Reconstruction parameters for TPC, no bool in here !!! BeginSubConfig(GPUSettingsRecTPC, tpc, configStandalone.rec, "RECTPC", 0, "Reconstruction settings", rec_tpc) -AddOptionRTC(rejectQPtB5, float, 1.f / 0.050f, "", 0, "QPt threshold to reject clusters of TPC tracks (Inverse Pt, scaled to B=0.5T!!!)") +AddOptionRTC(rejectQPtB5, float, 1.f / 0.050f, "", 0, "QPt threshold to reject clusters of TPC tracks (Inverse Pt, scaled to B=0.5T!!!)") // TODO: Sort these options automatically for parameter size AddOptionRTC(hitPickUpFactor, float, 1.f, "", 0, "multiplier for the combined cluster+track error during track following") AddOptionRTC(hitSearchArea2, float, 2.f, "", 0, "square of maximum search road of hits during seeding") AddOptionRTC(neighboursSearchArea, float, 3.f, "", 0, "area in cm for the search of neighbours, for z only used if searchWindowDZDR = 0") @@ -304,6 +304,14 @@ AddOption(nnCCDBInteractionRate, std::string, "500", "", 0, "Distinguishes betwe AddHelp("help", 'h') EndConfig() +// Debug Settings +BeginSubConfig(GPUSettingsProcessingDebug, debug, configStandalone.proc, "DEBUG", 0, "Debugging Settings", proc_debug) +AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1)) +AddOption(mergerMCLabels, bool, false, "", 0, "Create MC labels for merged tracks before refit for debugging") +AddHelp("help", 'h') +EndConfig() + + // Settings steering the processing once the device was selected, only available on the host BeginSubConfig(GPUSettingsProcessing, proc, configStandalone, "PROC", 0, "Processing settings", proc) AddOption(deviceNum, int32_t, -1, "gpuDevice", 0, "Set GPU device to use (-1: automatic, -2: for round-robin usage in timeslice-pipeline)") @@ -329,7 +337,6 @@ AddOption(memoryAllocationStrategy, int8_t, 0, "", 0, "Memory Allocation Strageg AddOption(forceMemoryPoolSize, uint64_t, 1, "memSize", 0, "Force size of allocated GPU / page locked host memory", min(0ul)) AddOption(forceHostMemoryPoolSize, uint64_t, 0, "hostMemSize", 0, "Force size of allocated host page locked host memory (overriding memSize)", min(0ul)) AddOption(memoryScalingFactor, float, 1.f, "", 0, "Factor to apply to all memory scalers") -AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1)) AddOption(conservativeMemoryEstimate, bool, false, "", 0, "Use some more conservative defaults for larger buffers during TPC processing") AddOption(tpcInputWithClusterRejection, uint8_t, 0, "", 0, "Indicate whether the TPC input is CTF data with cluster rejection, to tune buffer estimations") AddOption(forceMaxMemScalers, uint64_t, 0, "", 0, "Force using the maximum values for all buffers, Set a value n > 1 to rescale all maximums to a memory size of n") @@ -377,7 +384,7 @@ AddOption(tpcUseOldCPUDecoding, bool, false, "", 0, "Enable old CPU-based TPC de AddOption(tpcApplyCFCutsAtDecoding, bool, false, "", 0, "Apply cluster cuts from clusterization during decoding of compressed clusters") AddOption(tpcApplyClusterFilterOnCPU, uint8_t, 0, "", 0, "Apply custom cluster filter of GPUTPCClusterFilter class, 0: off, 1: debug, 2: PbPb23") AddOption(tpcWriteClustersAfterRejection, bool, false, "", 0, "Apply TPC rejection strategy before writing clusters") -AddOption(oclPlatformNum, int32_t, -1, "", 0, "Platform to use, in case the backend provides multiple platforms (OpenCL only, -1 = auto-select, -2 query all platforms (also incompatible))") +AddOption(oclPlatformNum, int32_t, -1, "", 0, "Platform to use, in case the backend provides multiple platforms (OpenCL only, -1 = auto-select, -2 query all platforms (also incompatible))") // TODO: Create some backend-specific options AddOption(oclCompileFromSources, bool, false, "", 0, "Compile OpenCL binary from included source code instead of using included spirv code") AddOption(oclOverrideSourceBuildFlags, std::string, "", "", 0, "Override OCL build flags for compilation from source, put a space for empty options") AddOption(printSettings, bool, false, "", 0, "Print all settings when initializing") @@ -396,6 +403,7 @@ AddSubConfig(GPUSettingsProcessingRTC, rtc) AddSubConfig(GPUSettingsProcessingRTCtechnical, rtctech) AddSubConfig(GPUSettingsProcessingParam, param) AddSubConfig(GPUSettingsProcessingNNclusterizer, nn) +AddSubConfig(GPUSettingsProcessingDebug, debug) AddHelp("help", 'h') EndConfig() #endif // __OPENCL__ diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 89a766c094df1..ad3ae802a5dd6 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -170,6 +170,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) runKernel({{1, -WarpSize(), 0, deviceType}}, 1); runKernel({{1, -WarpSize(), 0, deviceType}}, 1); } + if (!doGPU && GetProcessingSettings().debug.mergerMCLabels) { + Merger.CreateMCLabels(1, 1, 0, 0); + } DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingCollectedTracks, doGPU, Merger, &GPUTPCGMMerger::DumpCollected, *mDebugFile); if (param().rec.tpc.mergeCE) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 451ad53bc79d9..0582aa93ac6cd 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -148,6 +148,7 @@ using namespace o2::gpu::internal; #include "GPUQA.h" #include "GPUMemorySizeScalers.h" +#include "GPUQAHelper.h" GPUTPCGMMerger::GPUTPCGMMerger() { @@ -164,7 +165,7 @@ GPUTPCGMMerger::GPUTPCGMMerger() } // DEBUG CODE -#if !defined(GPUCA_GPUCODE) && (defined(GPUCA_MERGER_BY_MC_LABEL) || defined(GPUCA_CADEBUG_ENABLED) || GPUCA_MERGE_LOOPER_MC) +#if defined(GPUCA_MERGER_BY_MC_LABEL) || defined(GPUCA_CADEBUG_ENABLED) || GPUCA_MERGE_LOOPER_MC #include "GPUQAHelper.h" template @@ -438,6 +439,9 @@ void* GPUTPCGMMerger::SetPointersRefitScratch(void* mem) void* GPUTPCGMMerger::SetPointersOutput(void* mem) { computePointerWithAlignment(mem, mMergedTracks, mNMaxTracks); + if (mRec->GetProcessingSettings().debug.mergerMCLabels) { + computePointerWithAlignment(mem, mMergedTrackMC, mNMaxTracks); + } if (mRec->GetParam().dodEdxEnabled) { computePointerWithAlignment(mem, mMergedTracksdEdx, mNMaxTracks); if (mRec->GetParam().rec.tpc.dEdxClusterRejectionFlagMask != mRec->GetParam().rec.tpc.dEdxClusterRejectionFlagMaskAlt) { @@ -547,6 +551,34 @@ int32_t GPUTPCGMMerger::CheckSectors() return 0; } +void GPUTPCGMMerger::CreateMCLabels(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + const o2::tpc::ClusterNativeAccess* GPUrestrict() clusters = GetConstantMem()->ioPtrs.clustersNative; + if (clusters == nullptr || clusters->clustersMCTruth == nullptr) { + return; + } + if (mMergedTrackMC == nullptr) { + return; + } + + auto labelAssigner = GPUTPCTrkLbl(clusters->clustersMCTruth, 0.1f); + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + const auto& trk = mMergedTracks[i]; + if (!trk.OK()) { + continue; + } + labelAssigner.reset(); + for (uint32_t j = 0; j < trk.NClusters(); j++) { + const auto& cl = mClusters[trk.FirstClusterRef() + j]; + if (cl.state & GPUTPCGMMergedTrackHit::flagReject) { + continue; + } + labelAssigner.addLabel(cl.num); + } + mMergedTrackMC[i] = labelAssigner.computeLabel(); + } +} + #endif // GPUCA_GPUCODE GPUd() void GPUTPCGMMerger::ClearTrackLinks(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, bool output) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 6ee164435c81e..f0d19578913cd 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -113,6 +113,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() int32_t NMergedTracks() const { return mMemory->nMergedTracks; } GPUhdi() const GPUTPCGMMergedTrack* MergedTracks() const { return mMergedTracks; } + GPUhdi() const o2::MCCompLabel* MergedTrackMC() const { return mMergedTrackMC; } GPUhdi() GPUTPCGMMergedTrack* MergedTracks() { return mMergedTracks; } GPUhdi() const GPUdEdxInfo* MergedTracksdEdx() const { return mMergedTracksdEdx; } GPUhdi() GPUdEdxInfo* MergedTracksdEdx() { return mMergedTracksdEdx; } @@ -204,6 +205,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); GPUd() void ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void CreateMCLabels(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); #ifndef GPUCA_GPUCODE void DumpSectorTracks(std::ostream& out) const; @@ -284,6 +286,7 @@ class GPUTPCGMMerger : public GPUProcessor int32_t mNSectorHits = 0; // Total number of incoming clusters (from sector tracks) GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks + o2::MCCompLabel* mMergedTrackMC = nullptr; trackCluster* mClusterCandidates = nullptr; trackRebuildHelper* mTrackRebuildHelper = nullptr; int32_t* mHitWeights = nullptr; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 12cfcf9d6e330..839667378d0eb 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -14,6 +14,7 @@ #define GPUCA_CADEBUG 0 #define DEBUG_SINGLE_TRACK -1 +// #define DEBUG_REBUILD_MC #include "GPUTPCDef.h" #include "GPUTPCGMTrackParam.h" @@ -40,6 +41,11 @@ #include "AliHLTTPCClusterMCData.h" #endif +#ifndef GPUCA_GPUCODE +#include "SimulationDataFormat/ConstMCTruthContainer.h" +#include "SimulationDataFormat/MCCompLabel.h" +#endif + #ifndef GPUCA_GPUCODE_DEVICE #include #include @@ -279,6 +285,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && iWay >= nWays - 3 && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; int32_t sector = lastSector; + CADEBUG(merger.MergedTrackMC() printf("Extrapolate Start Track %d - sector %2d row %3d %s - fake %d\n", iTrk, sector, (int32_t)lastPropagateRow, up == 1 ? "upwards" : "downwards", (int)merger.MergedTrackMC()[iTrk].isFake())); uint8_t rowGapActive = 0, rowGapTotal = 0, missingRowsTotal = 0; uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow; uint8_t consecGoodRows = param.rec.tpc.rebuildTrackExtrMinConsecGoodRows, consecGoodRowsMissing = 0; @@ -327,6 +334,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ auto& candidate = merger.ClusterCandidates()[(iTrk * GPUCA_ROW_COUNT + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; if (candidate.id >= 2) { lastExtrapolateRow = iRow; +#if defined(DEBUG_REBUILD_MC) && !defined(GPUCA_GPUCODE) + if (merger.MergedTrackMC() && merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth) { + int32_t labelCorrect = GPUTPCTrkLblSearch(merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth->getLabels(candidate.id - 2), merger.MergedTrackMC()[iTrk]); + CADEBUG(printf("\t%21sLabel correct: %d\n", "", labelCorrect)); + } +#endif float err2Y, err2Z, xx, yy, zz; const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[candidate.id - 2]; merger.GetConstantMem()->calibObjects.fastTransformHelper->Transform(sector, iRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); diff --git a/GPU/GPUTracking/qa/GPUQAHelper.h b/GPU/GPUTracking/qa/GPUQAHelper.h index e9d98f3e4e305..14692cd5693b6 100644 --- a/GPU/GPUTracking/qa/GPUQAHelper.h +++ b/GPU/GPUTracking/qa/GPUQAHelper.h @@ -162,6 +162,17 @@ static inline auto GPUTPCTrkLbl(const AliHLTTPCClusterMCLabel* x, Args... args) } } +template +static inline bool GPUTPCTrkLblSearch(const T& clusterLabels, const MCCompLabel& trkLabel) +{ + for (const auto& clLabel : clusterLabels) { + if (trkLabel.compare(clLabel) >= 0) { + return true; + } + } + return false; +} + } // namespace gpu } // namespace o2