Skip to content

Commit d966c19

Browse files
committed
GPU TPC: Refactor rebuild data structures to remove GPU memory usage
1 parent 68affbc commit d966c19

7 files changed

Lines changed: 165 additions & 71 deletions

File tree

‎GPU/GPUTracking/Global/GPUChainTracking.cxx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -258,8 +258,8 @@ bool GPUChainTracking::ValidateSteps()
258258
bool GPUChainTracking::ValidateSettings()
259259
{
260260
int32_t gatherMode = mRec->GetProcessingSettings().tpcCompressionGatherMode == -1 ? mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCCompression).par_COMP_GATHER_MODE : mRec->GetProcessingSettings().tpcCompressionGatherMode;
261-
if (param().rec.tpc.rebuildTrackInFit && param().rec.tpc.rebuildTrackInFitClusterCandidates == 0) {
262-
GPUError("Need at least 1 cluster candidate to rebuild tracks during fit");
261+
if (param().rec.tpc.rebuildTrackInFit && (param().rec.tpc.rebuildTrackInFitClusterCandidates == 0 || param().rec.tpc.rebuildTrackInFitClusterCandidates > 13)) {
262+
GPUError("Need between 1 and 13 cluster candidates to rebuild tracks during fit");
263263
return false;
264264
}
265265
if (param().continuousMaxTimeBin > (int32_t)GPUSettings::TPC_MAX_TF_TIME_BIN) {

‎GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include "GPUO2DataTypes.h"
2020
#include "GPUQA.h"
2121
#include "GPUTPCGMMerger.h"
22+
#include "GPUTPCGMMergerRebuildInternal.h"
2223
#include "GPUConstantMem.h"
2324
#include "GPUTPCGMMergerGPU.h"
2425
#include "GPUTPCGMO2Output.h"
@@ -233,7 +234,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
233234
}
234235

235236
if (param().rec.tpc.rebuildTrackInFit) {
237+
AllocateRegisteredMemory(Merger.MemoryResRebuild()); // Sized by the number of merged tracks, which was transferred to the host above
238+
WriteToConstantMemory(RecoStep::TPCMerging, (char*)&processors()->tpcMerger - (char*)processors(), &MergerShadow, sizeof(MergerShadow), 0);
236239
runKernel<GPUMemClean16>({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates()));
240+
runKernel<GPUMemClean16>({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TrackRebuildHelperRow(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * sizeof(*MergerShadow.TrackRebuildHelperRow()));
237241
}
238242
// mRec->SetNActiveThreads(1); // For GPUTPCGMTrackParam:: GPUCA_CADEBUG: run the host kernels from here on with 1 thread, reset below
239243
runKernel<GPUTPCGMMergerTrackFit, GPUTPCGMMergerTrackFit::def>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0));

‎GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx‎

Lines changed: 57 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include "GPUTPCTracker.h"
3535
#include "GPUTPCTrackParam.h"
3636
#include "GPUTPCGMMerger.h"
37+
#include "GPUTPCGMMergerRebuildInternal.h"
3738
#include "GPUO2DataTypes.h"
3839
#include "TPCFastTransformPOD.h"
3940
#include "GPUTPCConvertImpl.h"
@@ -407,8 +408,6 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem)
407408
computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks does not save memory, other parts are larger anyway
408409
if (mRec->GetParam().rec.tpc.rebuildTrackInFit) {
409410
computePointerWithAlignment(mem, mSharedCount2, mNClusters);
410-
computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates);
411-
computePointerWithAlignment(mem, mTrackRebuildHelper, mNMaxTracks);
412411
computePointerWithAlignment(mem, mHitWeights, mNClusters);
413412
}
414413
if (mRec->GetParam().rec.tpc.retryRefit) {
@@ -427,6 +426,15 @@ void* GPUTPCGMMerger::SetPointersMemory(void* mem)
427426
return mem;
428427
}
429428

429+
void* GPUTPCGMMerger::SetPointersRebuild(void* mem)
430+
{
431+
const size_t nTracks = HostProcessor(this).NMergedTracks(); // Allocated once the number of merged tracks is known on the host
432+
computePointerWithAlignment(mem, mClusterCandidates, nTracks * GPUTPCGeometry::NROWS * mRec->GetParam().rec.tpc.rebuildTrackInFitClusterCandidates);
433+
computePointerWithAlignment(mem, mTrackRebuildHelperRow, nTracks * GPUTPCGeometry::NROWS);
434+
computePointerWithAlignment(mem, mTrackRebuildHelper, nTracks);
435+
return mem;
436+
}
437+
430438
void* GPUTPCGMMerger::SetPointersRefitScratch(void* mem)
431439
{
432440
computePointerWithAlignment(mem, mTrackOrderAttach, mNMaxTracks);
@@ -503,6 +511,7 @@ void GPUTPCGMMerger::RegisterMemoryAllocation()
503511
mMemoryResOutputO2MC = mRec->RegisterMemoryAllocation(this, &GPUTPCGMMerger::SetPointersOutputO2MC, GPUMemoryResource::MEMORY_OUTPUT_FLAG | GPUMemoryResource::MEMORY_HOST | GPUMemoryResource::MEMORY_CUSTOM, "TPCMergerOutputO2MC");
504512
}
505513
}
514+
mMemoryResRebuild = mRec->RegisterMemoryAllocation(this, &GPUTPCGMMerger::SetPointersRebuild, GPUMemoryResource::MEMORY_SCRATCH | GPUMemoryResource::MEMORY_STACK | GPUMemoryResource::MEMORY_CUSTOM, "TPCMergerRebuild");
506515
mMemoryResMemory = mRec->RegisterMemoryAllocation(this, &GPUTPCGMMerger::SetPointersMemory, GPUMemoryResource::MEMORY_PERMANENT, "TPCMergerMemory");
507516
}
508517

@@ -1697,7 +1706,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread
16971706
const GPUTPCTracker& trk = GetConstantMem()->tpcTrackers[t->Sector()];
16981707
const GPUTPCHitId& ic = trk.TrackHits()[t->OrigTrack()->FirstHitID() + i];
16991708
uint32_t id = trk.Data().ClusterDataIndex(trk.Data().Row(ic.RowIndex()), ic.HitIndex()) + GetConstantMem()->ioPtrs.clustersNative->clusterOffset[t->Sector()][0];
1700-
*c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f, .weight = 0, .best = 0};
1709+
*c2 = trackCluster{id, (uint8_t)ic.RowIndex(), t->Sector()};
17011710
}
17021711
nHits += nTrackHits;
17031712
}
@@ -2166,6 +2175,16 @@ GPUd() uint32_t GPUTPCGMMerger::MinHitsForQPt(float qPt, float err2QPt) const
21662175
return Param().tpcMinHitsB5Abs((CAMath::Abs(qPt) + CAMath::Min(CAMath::Sqrt(CAMath::Max(0.f, err2QPt)), Param().rec.tpc.trackFitMinHitsMaxQPtIncrease * CAMath::Abs(qPt))) * Param().qptB5Scaler);
21672176
}
21682177

2178+
GPUd() uint32_t GPUTPCGMMerger::CandidateId(uint32_t clusterId, uint8_t sector, uint8_t row) const
2179+
{
2180+
return clusterId - GetConstantMem()->ioPtrs.clustersNative->clusterOffset[sector][row];
2181+
}
2182+
2183+
GPUd() uint32_t GPUTPCGMMerger::CandidateClusterId(trackRebuildHelperCandidate candidate, uint8_t row) const
2184+
{
2185+
return GetConstantMem()->ioPtrs.clustersNative->clusterOffset[candidate.getSector()][row] + candidate.getId();
2186+
}
2187+
21692188
GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread)
21702189
{
21712190
for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) {
@@ -2186,10 +2205,10 @@ GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads,
21862205
for (uint32_t j = 0; j < trk.NClusters(); j++) {
21872206
const auto& cl = mClusters[trk.FirstClusterRef() + j];
21882207
auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * nMaxCandidates];
2189-
if ((!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0) && (candidates[0].id == 0 || cl.row == lastRow)) { // Row not searched: the original clusters of the row compete as normal candidates with fixed error
2208+
if ((!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0) && (candidates[0].isEmpty() || cl.row == lastRow)) { // Row not searched: the original clusters of the row compete as normal candidates with fixed error
21902209
for (uint32_t k = 0; k < nMaxCandidates; k++) {
2191-
if (candidates[k].id == 0) {
2192-
candidates[k] = {.id = cl.num + 2, .row = cl.row, .sector = cl.sector, .error = Param().rec.tpc.rebuildTrackFallbackErr, .weight = 0, .best = 0};
2210+
if (candidates[k].isEmpty()) {
2211+
candidates[k].set(CandidateId(cl.num, cl.sector, cl.row), cl.sector, (CAMath::half_t)Param().rec.tpc.rebuildTrackFallbackErr);
21932212
break;
21942213
}
21952214
}
@@ -2208,16 +2227,17 @@ GPUd() void GPUTPCGMMerger::ComputeHitWeights(int32_t nBlocks, int32_t nThreads,
22082227
continue;
22092228
}
22102229
for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) {
2211-
auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates];
2212-
if (candidates[0].best == 0) {
2213-
if (candidates[iteration].id >= 2 && (iteration == 0 || candidates[iteration].error <= CAMath::Max(Param().rec.tpc.rebuildTrackAltCandidateGoodErr, candidates[0].error * Param().rec.tpc.rebuildTrackAltCandidateMaxErrFactor))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster
2214-
float weightf = (trk.GetParam().GetNDF() > 0 ? (trk.GetParam().GetNDF() + 5) * trk.GetParam().GetNDF() * 64 / trk.GetParam().GetChi2() : (2 * (int32_t)trk.NClusters() - 5) * 128 / Param().rec.tpc.rebuildTrackFallbackTrackChi2) / (1.f + Param().rec.tpc.rebuildErrorWeightFactor * candidates[iteration].error); // Tracks with NDF <= 0 use their nominal number of clusters and assume a fixed chi2 per cluster
2215-
int32_t weight = weightf > 0.f ? (weightf > 2e9f ? 2e9f : (int32_t)weightf) : 0;
2216-
candidates[iteration].weight = weight;
2217-
CAMath::AtomicMax(&mHitWeights[candidates[iteration].id - 2], weight);
2218-
CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, candidates[iteration].id - 2, weight));
2230+
auto* c = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates];
2231+
auto& helper = mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j];
2232+
if (helper.getBest() == 0) {
2233+
if (c[iteration].hasCluster() && (iteration == 0 || c[iteration].error <= CAMath::Max<CAMath::half_t>((CAMath::half_t)Param().rec.tpc.rebuildTrackAltCandidateGoodErr, c[0].error * (CAMath::half_t)Param().rec.tpc.rebuildTrackAltCandidateMaxErrFactor))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster
2234+
float weightf = (trk.GetParam().GetNDF() > 0 ? (trk.GetParam().GetNDF() + 5) * trk.GetParam().GetNDF() * 64 / trk.GetParam().GetChi2() : (2 * (int32_t)trk.NClusters() - 5) * 128 / Param().rec.tpc.rebuildTrackFallbackTrackChi2); // Nominal number of clusters and fixed chi2 per cluster for NDF <= 0
2235+
weightf /= (1.f + Param().rec.tpc.rebuildErrorWeightFactor * (float)c[iteration].error);
2236+
const int32_t weight = helper.setWeight(weightf);
2237+
CAMath::AtomicMax(&mHitWeights[CandidateClusterId(c[iteration], j)], weight);
2238+
CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, CandidateClusterId(c[iteration], j), weight));
22192239
} else {
2220-
candidates[0].best = -2;
2240+
helper.setBest(trackRebuildHelperRow::bestNoCandidate);
22212241
}
22222242
}
22232243
}
@@ -2233,10 +2253,9 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights1(int32_t nBlocks, int32_t nThreads
22332253
}
22342254
for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) {
22352255
auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates];
2236-
if (candidates[0].best == 0) {
2237-
if (candidates[iteration].weight == mHitWeights[candidates[iteration].id - 2]) {
2238-
candidates[0].best = iteration + 1;
2239-
}
2256+
auto& helper = mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j];
2257+
if (helper.getBest() == 0 && helper.getWeight() == mHitWeights[CandidateClusterId(candidates[iteration], j)]) {
2258+
helper.setBest(iteration + 1);
22402259
}
22412260
}
22422261
}
@@ -2256,15 +2275,17 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads
22562275
uint8_t nSharedRowCandidates = 0;
22572276
for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) {
22582277
auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates];
2259-
if (candidates[0].best == 0 && candidates[nIter - 1].weight == mHitWeights[candidates[nIter - 1].id - 2]) {
2260-
candidates[0].best = nIter;
2278+
auto& helper = mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j]; // the helper weight is from the last computed iteration: the one that set best, or the last one if best is still 0
2279+
if (helper.getBest() == 0 && helper.getWeight() == mHitWeights[CandidateClusterId(candidates[nIter - 1], j)]) {
2280+
helper.setBest(nIter);
22612281
}
2262-
if (candidates[0].best > 0) {
2282+
if (helper.getBest() > 0 && helper.getBest() < trackRebuildHelperRow::bestRowRemoved) {
22632283
nHits++;
2264-
if (candidates[candidates[0].best - 1].weight != mHitWeights[candidates[candidates[0].best - 1].id - 2]) {
2265-
mSharedCount2[candidates[candidates[0].best - 1].id - 2] = 1; // shared between tracks of different iterations
2284+
const uint32_t id = CandidateClusterId(candidates[helper.getBest() - 1], j);
2285+
if (helper.getWeight() != mHitWeights[id]) {
2286+
mSharedCount2[id] = 1; // shared between tracks of different iterations
22662287
}
2267-
} else if (candidates[0].best != -1 && candidates[0].id >= 2) {
2288+
} else if (helper.getBest() != trackRebuildHelperRow::bestRowRemoved && candidates[0].hasCluster()) {
22682289
sharedRowCandidates[nSharedRowCandidates++] = j;
22692290
}
22702291
}
@@ -2274,16 +2295,16 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads
22742295
if (nShared) {
22752296
const uint32_t n = Param().rec.tpc.rebuildTrackInFitClusterCandidates;
22762297
const auto* candidates = &mClusterCandidates[i * GPUTPCGeometry::NROWS * n];
2277-
GPUCommonAlgorithm::sort(sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [n, candidates](const uint8_t& a, const uint8_t& b) {
2298+
GPUCommonAlgorithm::sort(sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [n, candidates, this](const uint8_t& a, const uint8_t& b) {
22782299
const auto& ca = candidates[a * n];
22792300
const auto& cb = candidates[b * n];
2280-
return GPUCA_DETERMINISTIC_CODE(ca.error == cb.error ? ca.id < cb.id : ca.error < cb.error, ca.error < cb.error);
2301+
return GPUCA_DETERMINISTIC_CODE(ca.error == cb.error ? CandidateClusterId(ca, a) < CandidateClusterId(cb, b) : ca.error < cb.error, ca.error < cb.error);
22812302
});
22822303
for (uint32_t j = 0; j < nShared; j++) {
22832304
auto* cj = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + sharedRowCandidates[j]) * Param().rec.tpc.rebuildTrackInFitClusterCandidates];
2284-
CADEBUG(printf("REBUILD: iTrk %d Recovering shared %d / %d: row %d %d, error %f\n", i, j, nShared, (int)sharedRowCandidates[j], cj[0].id - 2, cj[0].error));
2285-
cj[0].best = 1;
2286-
mSharedCount2[cj[0].id - 2] = 1; // TODO: Should we also recover iter0 clusters?
2305+
CADEBUG(printf("REBUILD: iTrk %d Recovering shared %d / %d: row %d %d, error %f\n", i, j, nShared, (int)sharedRowCandidates[j], CandidateClusterId(cj[0], sharedRowCandidates[j]), cj[0].error));
2306+
mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + sharedRowCandidates[j]].setBest(1);
2307+
mSharedCount2[CandidateClusterId(cj[0], sharedRowCandidates[j])] = 1; // TODO: Should we also recover iter0 clusters?
22872308
}
22882309
nHits += nShared;
22892310
}
@@ -2304,17 +2325,18 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads
23042325
const bool reverse = mTrackRebuildHelper[i].reverse; // TODO: CE tracks may have clusters on both sides in the rows around the crossing: make sure the track crosses the CE only once, by cutting at the best row and removing the clusters on the wrong side of the cut
23052326
for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) {
23062327
const auto& candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates];
2307-
if (candidates[0].best > 0) {
2328+
if (mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() > 0 && mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() < trackRebuildHelperRow::bestRowRemoved) {
23082329
auto& outCl = mClusters[iOut + (reverse ? (written) : (nHits - 1 - written))];
2309-
const auto& best = candidates[candidates[0].best - 1];
2310-
const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[best.id - 2];
2330+
const auto& best = candidates[mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() - 1];
2331+
const uint32_t id = CandidateClusterId(best, j);
2332+
const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[id];
23112333
uint8_t flags = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags);
23122334
if ((mTrackRebuildHelper[i].highInclRowLow != 255 && j <= mTrackRebuildHelper[i].highInclRowLow) || (mTrackRebuildHelper[i].highInclRowHigh != 255 && j >= mTrackRebuildHelper[i].highInclRowHigh)) {
23132335
flags |= GPUTPCGMMergedTrackHit::flagHighIncl;
23142336
}
2315-
outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = flags};
2337+
outCl = {.num = id, .sector = best.getSector(), .row = (uint8_t)j, .state = flags};
23162338
written++;
2317-
CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, best.id - 2));
2339+
CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, id));
23182340
}
23192341
}
23202342
}

0 commit comments

Comments
 (0)