3434#include " GPUTPCTracker.h"
3535#include " GPUTPCTrackParam.h"
3636#include " GPUTPCGMMerger.h"
37+ #include " GPUTPCGMMergerRebuildInternal.h"
3738#include " GPUO2DataTypes.h"
3839#include " TPCFastTransformPOD.h"
3940#include " GPUTPCConvertImpl.h"
@@ -407,8 +408,6 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem)
407408 computePointerWithAlignment (mem, mLoopData , mNMaxTracks ); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks does not save memory, other parts are larger anyway
408409 if (mRec ->GetParam ().rec .tpc .rebuildTrackInFit ) {
409410 computePointerWithAlignment (mem, mSharedCount2 , mNClusters );
410- computePointerWithAlignment (mem, mClusterCandidates , mNMaxTracks * GPUTPCGeometry::NROWS * Param ().rec .tpc .rebuildTrackInFitClusterCandidates );
411- computePointerWithAlignment (mem, mTrackRebuildHelper , mNMaxTracks );
412411 computePointerWithAlignment (mem, mHitWeights , mNClusters );
413412 }
414413 if (mRec ->GetParam ().rec .tpc .retryRefit ) {
@@ -427,6 +426,15 @@ void* GPUTPCGMMerger::SetPointersMemory(void* mem)
427426 return mem;
428427}
429428
429+ void * GPUTPCGMMerger::SetPointersRebuild (void * mem)
430+ {
431+ const size_t nTracks = HostProcessor (this ).NMergedTracks (); // Allocated once the number of merged tracks is known on the host
432+ computePointerWithAlignment (mem, mClusterCandidates , nTracks * GPUTPCGeometry::NROWS * mRec ->GetParam ().rec .tpc .rebuildTrackInFitClusterCandidates );
433+ computePointerWithAlignment (mem, mTrackRebuildHelperRow , nTracks * GPUTPCGeometry::NROWS );
434+ computePointerWithAlignment (mem, mTrackRebuildHelper , nTracks);
435+ return mem;
436+ }
437+
430438void * GPUTPCGMMerger::SetPointersRefitScratch (void * mem)
431439{
432440 computePointerWithAlignment (mem, mTrackOrderAttach , mNMaxTracks );
@@ -503,6 +511,7 @@ void GPUTPCGMMerger::RegisterMemoryAllocation()
503511 mMemoryResOutputO2MC = mRec ->RegisterMemoryAllocation (this , &GPUTPCGMMerger::SetPointersOutputO2MC, GPUMemoryResource::MEMORY_OUTPUT_FLAG | GPUMemoryResource::MEMORY_HOST | GPUMemoryResource::MEMORY_CUSTOM , " TPCMergerOutputO2MC" );
504512 }
505513 }
514+ mMemoryResRebuild = mRec ->RegisterMemoryAllocation (this , &GPUTPCGMMerger::SetPointersRebuild, GPUMemoryResource::MEMORY_SCRATCH | GPUMemoryResource::MEMORY_STACK | GPUMemoryResource::MEMORY_CUSTOM , " TPCMergerRebuild" );
506515 mMemoryResMemory = mRec ->RegisterMemoryAllocation (this , &GPUTPCGMMerger::SetPointersMemory, GPUMemoryResource::MEMORY_PERMANENT , " TPCMergerMemory" );
507516}
508517
@@ -1697,7 +1706,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread
16971706 const GPUTPCTracker& trk = GetConstantMem ()->tpcTrackers [t->Sector ()];
16981707 const GPUTPCHitId& ic = trk.TrackHits ()[t->OrigTrack ()->FirstHitID () + i];
16991708 uint32_t id = trk.Data ().ClusterDataIndex (trk.Data ().Row (ic.RowIndex ()), ic.HitIndex ()) + GetConstantMem ()->ioPtrs .clustersNative ->clusterOffset [t->Sector ()][0 ];
1700- *c2 = trackCluster{. id = id, . row = (uint8_t )ic.RowIndex (), . sector = t->Sector (), . error = 0 . f , . weight = 0 , . best = 0 };
1709+ *c2 = trackCluster{id, (uint8_t )ic.RowIndex (), t->Sector ()};
17011710 }
17021711 nHits += nTrackHits;
17031712 }
@@ -2166,6 +2175,16 @@ GPUd() uint32_t GPUTPCGMMerger::MinHitsForQPt(float qPt, float err2QPt) const
21662175 return Param ().tpcMinHitsB5Abs ((CAMath::Abs (qPt) + CAMath::Min (CAMath::Sqrt (CAMath::Max (0 .f , err2QPt)), Param ().rec .tpc .trackFitMinHitsMaxQPtIncrease * CAMath::Abs (qPt))) * Param ().qptB5Scaler );
21672176}
21682177
2178+ GPUd () uint32_t GPUTPCGMMerger::CandidateId(uint32_t clusterId, uint8_t sector, uint8_t row) const
2179+ {
2180+ return clusterId - GetConstantMem ()->ioPtrs .clustersNative ->clusterOffset [sector][row];
2181+ }
2182+
2183+ GPUd () uint32_t GPUTPCGMMerger::CandidateClusterId(trackRebuildHelperCandidate candidate, uint8_t row) const
2184+ {
2185+ return GetConstantMem ()->ioPtrs .clustersNative ->clusterOffset [candidate.getSector ()][row] + candidate.getId ();
2186+ }
2187+
21692188GPUd () void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread)
21702189{
21712190 for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters ; i += nBlocks * nThreads) {
@@ -2186,10 +2205,10 @@ GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads,
21862205 for (uint32_t j = 0 ; j < trk.NClusters (); j++) {
21872206 const auto & cl = mClusters [trk.FirstClusterRef () + j];
21882207 auto * candidates = &mClusterCandidates [(i * GPUTPCGeometry::NROWS + cl.row ) * nMaxCandidates];
2189- if ((!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam ().GetNDF () <= 0 ) && (candidates[0 ].id == 0 || cl.row == lastRow)) { // Row not searched: the original clusters of the row compete as normal candidates with fixed error
2208+ if ((!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam ().GetNDF () <= 0 ) && (candidates[0 ].isEmpty () || cl.row == lastRow)) { // Row not searched: the original clusters of the row compete as normal candidates with fixed error
21902209 for (uint32_t k = 0 ; k < nMaxCandidates; k++) {
2191- if (candidates[k].id == 0 ) {
2192- candidates[k] = {. id = cl.num + 2 , . row = cl.row , . sector = cl.sector , . error = Param ().rec .tpc .rebuildTrackFallbackErr , . weight = 0 , . best = 0 } ;
2210+ if (candidates[k].isEmpty () ) {
2211+ candidates[k]. set ( CandidateId ( cl.num , cl. sector , cl.row ), cl.sector , (CAMath:: half_t ) Param ().rec .tpc .rebuildTrackFallbackErr ) ;
21932212 break ;
21942213 }
21952214 }
@@ -2208,16 +2227,17 @@ GPUd() void GPUTPCGMMerger::ComputeHitWeights(int32_t nBlocks, int32_t nThreads,
22082227 continue ;
22092228 }
22102229 for (uint32_t j = 0 ; j < GPUTPCGeometry::NROWS ; j++) {
2211- auto * candidates = &mClusterCandidates [(i * GPUTPCGeometry::NROWS + j) * Param ().rec .tpc .rebuildTrackInFitClusterCandidates ];
2212- if (candidates[0 ].best == 0 ) {
2213- if (candidates[iteration].id >= 2 && (iteration == 0 || candidates[iteration].error <= CAMath::Max (Param ().rec .tpc .rebuildTrackAltCandidateGoodErr , candidates[0 ].error * Param ().rec .tpc .rebuildTrackAltCandidateMaxErrFactor ))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster
2214- float weightf = (trk.GetParam ().GetNDF () > 0 ? (trk.GetParam ().GetNDF () + 5 ) * trk.GetParam ().GetNDF () * 64 / trk.GetParam ().GetChi2 () : (2 * (int32_t )trk.NClusters () - 5 ) * 128 / Param ().rec .tpc .rebuildTrackFallbackTrackChi2 ) / (1 .f + Param ().rec .tpc .rebuildErrorWeightFactor * candidates[iteration].error ); // Tracks with NDF <= 0 use their nominal number of clusters and assume a fixed chi2 per cluster
2215- int32_t weight = weightf > 0 .f ? (weightf > 2e9f ? 2e9f : (int32_t )weightf) : 0 ;
2216- candidates[iteration].weight = weight;
2217- CAMath::AtomicMax (&mHitWeights [candidates[iteration].id - 2 ], weight);
2218- CADEBUG (printf (" REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n " , i, iteration, j, candidates[iteration].id - 2 , weight));
2230+ auto * c = &mClusterCandidates [(i * GPUTPCGeometry::NROWS + j) * Param ().rec .tpc .rebuildTrackInFitClusterCandidates ];
2231+ auto & helper = mTrackRebuildHelperRow [i * GPUTPCGeometry::NROWS + j];
2232+ if (helper.getBest () == 0 ) {
2233+ if (c[iteration].hasCluster () && (iteration == 0 || c[iteration].error <= CAMath::Max<CAMath::half_t >((CAMath::half_t )Param ().rec .tpc .rebuildTrackAltCandidateGoodErr , c[0 ].error * (CAMath::half_t )Param ().rec .tpc .rebuildTrackAltCandidateMaxErrFactor ))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster
2234+ float weightf = (trk.GetParam ().GetNDF () > 0 ? (trk.GetParam ().GetNDF () + 5 ) * trk.GetParam ().GetNDF () * 64 / trk.GetParam ().GetChi2 () : (2 * (int32_t )trk.NClusters () - 5 ) * 128 / Param ().rec .tpc .rebuildTrackFallbackTrackChi2 ); // Nominal number of clusters and fixed chi2 per cluster for NDF <= 0
2235+ weightf /= (1 .f + Param ().rec .tpc .rebuildErrorWeightFactor * (float )c[iteration].error );
2236+ const int32_t weight = helper.setWeight (weightf);
2237+ CAMath::AtomicMax (&mHitWeights [CandidateClusterId (c[iteration], j)], weight);
2238+ CADEBUG (printf (" REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n " , i, iteration, j, CandidateClusterId (c[iteration], j), weight));
22192239 } else {
2220- candidates[ 0 ]. best = - 2 ;
2240+ helper. setBest (trackRebuildHelperRow::bestNoCandidate) ;
22212241 }
22222242 }
22232243 }
@@ -2233,10 +2253,9 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights1(int32_t nBlocks, int32_t nThreads
22332253 }
22342254 for (unsigned int j = 0 ; j < GPUTPCGeometry::NROWS ; j++) {
22352255 auto * candidates = &mClusterCandidates [(i * GPUTPCGeometry::NROWS + j) * Param ().rec .tpc .rebuildTrackInFitClusterCandidates ];
2236- if (candidates[0 ].best == 0 ) {
2237- if (candidates[iteration].weight == mHitWeights [candidates[iteration].id - 2 ]) {
2238- candidates[0 ].best = iteration + 1 ;
2239- }
2256+ auto & helper = mTrackRebuildHelperRow [i * GPUTPCGeometry::NROWS + j];
2257+ if (helper.getBest () == 0 && helper.getWeight () == mHitWeights [CandidateClusterId (candidates[iteration], j)]) {
2258+ helper.setBest (iteration + 1 );
22402259 }
22412260 }
22422261 }
@@ -2256,15 +2275,17 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads
22562275 uint8_t nSharedRowCandidates = 0 ;
22572276 for (unsigned int j = 0 ; j < GPUTPCGeometry::NROWS ; j++) {
22582277 auto * candidates = &mClusterCandidates [(i * GPUTPCGeometry::NROWS + j) * Param ().rec .tpc .rebuildTrackInFitClusterCandidates ];
2259- if (candidates[0 ].best == 0 && candidates[nIter - 1 ].weight == mHitWeights [candidates[nIter - 1 ].id - 2 ]) {
2260- candidates[0 ].best = nIter;
2278+ auto & helper = mTrackRebuildHelperRow [i * GPUTPCGeometry::NROWS + j]; // the helper weight is from the last computed iteration: the one that set best, or the last one if best is still 0
2279+ if (helper.getBest () == 0 && helper.getWeight () == mHitWeights [CandidateClusterId (candidates[nIter - 1 ], j)]) {
2280+ helper.setBest (nIter);
22612281 }
2262- if (candidates[ 0 ]. best > 0 ) {
2282+ if (helper. getBest () > 0 && helper. getBest () < trackRebuildHelperRow::bestRowRemoved ) {
22632283 nHits++;
2264- if (candidates[candidates[0 ].best - 1 ].weight != mHitWeights [candidates[candidates[0 ].best - 1 ].id - 2 ]) {
2265- mSharedCount2 [candidates[candidates[0 ].best - 1 ].id - 2 ] = 1 ; // shared between tracks of different iterations
2284+ const uint32_t id = CandidateClusterId (candidates[helper.getBest () - 1 ], j);
2285+ if (helper.getWeight () != mHitWeights [id]) {
2286+ mSharedCount2 [id] = 1 ; // shared between tracks of different iterations
22662287 }
2267- } else if (candidates[ 0 ]. best != - 1 && candidates[0 ].id >= 2 ) {
2288+ } else if (helper. getBest () != trackRebuildHelperRow::bestRowRemoved && candidates[0 ].hasCluster () ) {
22682289 sharedRowCandidates[nSharedRowCandidates++] = j;
22692290 }
22702291 }
@@ -2274,16 +2295,16 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads
22742295 if (nShared) {
22752296 const uint32_t n = Param ().rec .tpc .rebuildTrackInFitClusterCandidates ;
22762297 const auto * candidates = &mClusterCandidates [i * GPUTPCGeometry::NROWS * n];
2277- GPUCommonAlgorithm::sort (sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [n, candidates](const uint8_t & a, const uint8_t & b) {
2298+ GPUCommonAlgorithm::sort (sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [n, candidates, this ](const uint8_t & a, const uint8_t & b) {
22782299 const auto & ca = candidates[a * n];
22792300 const auto & cb = candidates[b * n];
2280- return GPUCA_DETERMINISTIC_CODE (ca.error == cb.error ? ca. id < cb. id : ca.error < cb.error , ca.error < cb.error );
2301+ return GPUCA_DETERMINISTIC_CODE (ca.error == cb.error ? CandidateClusterId (ca, a) < CandidateClusterId (cb, b) : ca.error < cb.error , ca.error < cb.error );
22812302 });
22822303 for (uint32_t j = 0 ; j < nShared; j++) {
22832304 auto * cj = &mClusterCandidates [(i * GPUTPCGeometry::NROWS + sharedRowCandidates[j]) * Param ().rec .tpc .rebuildTrackInFitClusterCandidates ];
2284- CADEBUG (printf (" REBUILD: iTrk %d Recovering shared %d / %d: row %d %d, error %f\n " , i, j, nShared, (int )sharedRowCandidates[j], cj[0 ]. id - 2 , cj[0 ].error ));
2285- cj[ 0 ]. best = 1 ;
2286- mSharedCount2 [cj[0 ]. id - 2 ] = 1 ; // TODO: Should we also recover iter0 clusters?
2305+ CADEBUG (printf (" REBUILD: iTrk %d Recovering shared %d / %d: row %d %d, error %f\n " , i, j, nShared, (int )sharedRowCandidates[j], CandidateClusterId ( cj[0 ], sharedRowCandidates[j]) , cj[0 ].error ));
2306+ mTrackRebuildHelperRow [i * GPUTPCGeometry:: NROWS + sharedRowCandidates[j]]. setBest ( 1 ) ;
2307+ mSharedCount2 [CandidateClusterId ( cj[0 ], sharedRowCandidates[j]) ] = 1 ; // TODO: Should we also recover iter0 clusters?
22872308 }
22882309 nHits += nShared;
22892310 }
@@ -2304,17 +2325,18 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads
23042325 const bool reverse = mTrackRebuildHelper [i].reverse ; // TODO: CE tracks may have clusters on both sides in the rows around the crossing: make sure the track crosses the CE only once, by cutting at the best row and removing the clusters on the wrong side of the cut
23052326 for (unsigned int j = 0 ; j < GPUTPCGeometry::NROWS ; j++) {
23062327 const auto & candidates = &mClusterCandidates [(i * GPUTPCGeometry::NROWS + j) * Param ().rec .tpc .rebuildTrackInFitClusterCandidates ];
2307- if (candidates[ 0 ]. best > 0 ) {
2328+ if (mTrackRebuildHelperRow [i * GPUTPCGeometry:: NROWS + j]. getBest () > 0 && mTrackRebuildHelperRow [i * GPUTPCGeometry:: NROWS + j]. getBest () < trackRebuildHelperRow::bestRowRemoved ) {
23082329 auto & outCl = mClusters [iOut + (reverse ? (written) : (nHits - 1 - written))];
2309- const auto & best = candidates[candidates[0 ].best - 1 ];
2310- const ClusterNative& GPUrestrict () cl = GetConstantMem ()->ioPtrs .clustersNative ->clustersLinear [best.id - 2 ];
2330+ const auto & best = candidates[mTrackRebuildHelperRow [i * GPUTPCGeometry::NROWS + j].getBest () - 1 ];
2331+ const uint32_t id = CandidateClusterId (best, j);
2332+ const ClusterNative& GPUrestrict () cl = GetConstantMem ()->ioPtrs .clustersNative ->clustersLinear [id];
23112333 uint8_t flags = (uint8_t )(cl.getFlags () & GPUTPCGMMergedTrackHit::clustererAndSharedFlags);
23122334 if ((mTrackRebuildHelper [i].highInclRowLow != 255 && j <= mTrackRebuildHelper [i].highInclRowLow ) || (mTrackRebuildHelper [i].highInclRowHigh != 255 && j >= mTrackRebuildHelper [i].highInclRowHigh )) {
23132335 flags |= GPUTPCGMMergedTrackHit::flagHighIncl;
23142336 }
2315- outCl = {.num = best. id - 2 , .sector = best.sector , .row = (uint8_t )j, .state = flags};
2337+ outCl = {.num = id , .sector = best.getSector () , .row = (uint8_t )j, .state = flags};
23162338 written++;
2317- CADEBUG (printf (" REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n " , i, j, best. id - 2 ));
2339+ CADEBUG (printf (" REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n " , i, j, id ));
23182340 }
23192341 }
23202342 }
0 commit comments