Skip to content

Commit 68affbc

Browse files
committed
GPU: Provide separate fit kernels for rebuild/retry, remove mergerSortTracks runtime option
1 parent 5b5d978 commit 68affbc

12 files changed

Lines changed: 67 additions & 25 deletions

‎GPU/GPUTracking/Base/GPUReconstruction.cxx‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -301,9 +301,6 @@ int32_t GPUReconstruction::InitPhaseBeforeDevice()
301301
if (!GetProcessingSettings().createO2Output || !IsGPU()) {
302302
mProcessingSettings->clearO2OutputFromGPU = false;
303303
}
304-
if (!(mRecoSteps.stepsGPUMask & gpudatatypes::RecoStep::TPCMerging)) {
305-
mProcessingSettings->mergerSortTracks = false;
306-
}
307304
if (GetProcessingSettings().debugLevel > 3 || !IsGPU() || GetProcessingSettings().deterministicGPUReconstruction) {
308305
mProcessingSettings->delayedOutput = false;
309306
}

‎GPU/GPUTracking/Definitions/GPUGetConstexpr.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222

2323
// This is a temporary workaround required for clang (with c++20), until we can go to C++23 with P2280R4, which allows getting constexpr static values from references
2424

25-
#if defined(__clang__) && __cplusplus >= 202002L && __cplusplus < 202302L
25+
#if (defined(__clang__) && __cplusplus >= 202002L && __cplusplus < 202302L) || defined(__CPUDEVICE__) // BUG: The workaround is needed for GCC if used inside a templated function.
2626

2727
namespace o2::gpu::internal
2828
{

‎GPU/GPUTracking/Definitions/GPUSettingsList.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,6 @@ AddOption(nStreams, int8_t, 8, "", 0, "Number of GPU streams / command queues")
397397
AddOption(nTPCClustererLanes, int8_t, -1, "", 0, "Number of TPC clusterers that can run in parallel (-1 = autoset)")
398398
AddOption(overrideClusterizerFragmentLen, int32_t, -1, "", 0, "Force the cluster max fragment len to a certain value (-1 = autodetect)")
399399
AddOption(delayedOutput, bool, true, "", 0, "Delay output to be parallel to track fit")
400-
AddOption(mergerSortTracks, int8_t, -1, "", 0, "Sort track indizes for GPU track fit (-1 = auto-device based on device parameters)")
401400
AddOption(alternateBorderSort, int8_t, -1, "", 0, "Alternative implementation for sorting of border tracks (-1 = auto-device based on device parameters)")
402401
AddOption(tpcCompressionGatherMode, int8_t, -1, "", 0, "TPC Compressed Clusters Gather Mode (0: DMA transfer gather gpu to host, 1: serial DMA to host and gather by copy on CPU, 2. gather via GPU kernal DMA access, 3. gather on GPU via kernel, dma afterwards, -1 = auto-device based on device parameters)")
403402
AddOption(tpcCompressionGatherModeKernel, int8_t, -1, "", 0, "TPC Compressed Clusters Gather Mode Kernel (0: unbufferd, 1-3: buffered, 4: multi-block, -1 = auto-device based on device parameters)")

‎GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@ GPUTPCCFDecodeZSLink,"""GPUCA_WARP_SIZE""",,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_
2727
GPUTPCCFDecodeZSDenseLink,"""GPUCA_WARP_SIZE""",,"[""GPUCA_WARP_SIZE"", 4]","[""GPUCA_WARP_SIZE"", 14]",,,,,,"[""GPUCA_WARP_SIZE"", 14]","""GPUCA_WARP_SIZE""","[""GPUCA_WARP_SIZE"", 22]","[""GPUCA_WARP_SIZE"", 22]",,"[""GPUCA_WARP_SIZE"", 17]",,"[""GPUCA_WARP_SIZE"", 8]","[""GPUCA_WARP_SIZE"", 5]",
2828
GPUTPCCFGather,"[1024, 1]",,"[1024, 5]","[1024, 1]",,,,,,"[160, 11]","[1024, 1]",736,896,,"[928, 1]",,"[1024, 1]","[320, 2]",
2929
COMPRESSION_GATHER,1024,,1024,1024,,,,,,1024,1024,,1024,,,,,,
30-
GPUTPCGMMergerTrackFit,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]","[64, 4]"
30+
GPUTPCGMMergerTrackFit_def,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]","[64, 4]"
31+
GPUTPCGMMergerTrackFit_rebuild,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]","[64, 4]"
32+
GPUTPCGMMergerTrackFit_retry,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]","[64, 4]"
3133
GPUTPCGMMergerFollowLoopers,256,,"[256, 5]","[256, 4]",,,,,,"[256, 4]","[128, 4]","[1024, 1]",640,,"[128, 16]",,"[224, 3]","[256, 7]","[32, 64]"
3234
GPUTPCGMMergerSectorRefit,256,,"[64, 4]","[256, 2]",,,,,,"[32, 8]","[64, 5]","[32, 7]","[32, 7]",,"[32, 20]",,"[32, 10]","[64, 4]","[256, 8]"
3335
GPUTPCGMMergerUnpackResetIds,256,,256,256,,,,,,256,256,256,256,,256,,256,256,

‎GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,21 @@
167167
"AMPERE": 1024,
168168
"TURING": 1024
169169
},
170-
"GPUTPCGMMergerTrackFit": {
170+
"GPUTPCGMMergerTrackFit_def": {
171+
"default": 256,
172+
"MI100": [192, 2],
173+
"VEGA": [64, 7],
174+
"AMPERE": [64, 4],
175+
"TURING": [32, 8]
176+
},
177+
"GPUTPCGMMergerTrackFit_rebuild": {
178+
"default": 256,
179+
"MI100": [192, 2],
180+
"VEGA": [64, 7],
181+
"AMPERE": [64, 4],
182+
"TURING": [32, 8]
183+
},
184+
"GPUTPCGMMergerTrackFit_retry": {
171185
"default": 256,
172186
"MI100": [192, 2],
173187
"VEGA": [64, 7],

‎GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
196196
waitForTransfer = 1;
197197
}
198198

199-
const bool mergerSortTracks = GetProcessingSettings().mergerSortTracks == -1 ? mRec->getGPUParameters(doGPU).par_SORT_BEFORE_FIT : GetProcessingSettings().mergerSortTracks;
199+
const bool mergerSortTracks = mRec->getGPUParameters(doGPU).par_SORT_BEFORE_FIT;
200200
if (mergerSortTracks) {
201201
runKernel<GPUTPCGMMergerSortTracksPrepare>(GetGridAuto(0, deviceType));
202202
CondWaitEvent(waitForTransfer, &mEvents->single);
@@ -236,7 +236,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
236236
runKernel<GPUMemClean16>({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates()));
237237
}
238238
// mRec->SetNActiveThreads(1); // For GPUTPCGMTrackParam:: GPUCA_CADEBUG: run the host kernels from here on with 1 thread, reset below
239-
runKernel<GPUTPCGMMergerTrackFit>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0);
239+
runKernel<GPUTPCGMMergerTrackFit, GPUTPCGMMergerTrackFit::def>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0));
240240
if (param().rec.tpc.rebuildTrackInFit) {
241241
runKernel<GPUMemClean16>({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.HitWeights(), Merger.NClusters() * sizeof(*MergerShadow.HitWeights()));
242242
runKernel<GPUTPCGMMergerHitWeights, GPUTPCGMMergerHitWeights::prepare>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0);
@@ -250,10 +250,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
250250
DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpInterpolatedHits, *mDebugFile);
251251
runKernel<GPUTPCGMMergerHitWeights, GPUTPCGMMergerHitWeights::resolveShared>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0);
252252
DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRebuiltTracks, *mDebugFile);
253-
runKernel<GPUTPCGMMergerTrackFit>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1);
253+
runKernel<GPUTPCGMMergerTrackFit, GPUTPCGMMergerTrackFit::rebuild>(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0));
254254
}
255255
if (param().rec.tpc.retryRefit) {
256-
runKernel<GPUTPCGMMergerTrackFit>(GetGridAuto(0), -1, param().rec.tpc.rebuildTrackInFit);
256+
runKernel<GPUTPCGMMergerTrackFit, GPUTPCGMMergerTrackFit::retry>(GetGridAuto(0));
257257
}
258258

259259
runKernel<GPUTPCGMMergerFollowLoopers>(GetGridAuto(0));

‎GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -430,8 +430,7 @@ void* GPUTPCGMMerger::SetPointersMemory(void* mem)
430430
void* GPUTPCGMMerger::SetPointersRefitScratch(void* mem)
431431
{
432432
computePointerWithAlignment(mem, mTrackOrderAttach, mNMaxTracks);
433-
const bool mergerSortTracks = mRec->GetProcessingSettings().mergerSortTracks == -1 ? mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCMerging).par_SORT_BEFORE_FIT : mRec->GetProcessingSettings().mergerSortTracks;
434-
if (mergerSortTracks) {
433+
if (mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCMerging).par_SORT_BEFORE_FIT) {
435434
computePointerWithAlignment(mem, mTrackOrderProcess, mNMaxTracks);
436435
}
437436
return mem;

‎GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,30 @@
1818

1919
using namespace o2::gpu;
2020

21-
template <>
22-
GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode, int32_t rebuilt)
21+
template <int32_t I>
22+
GPUdii() void GPUTPCGMMergerTrackFit::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger)
2323
{
24-
const int32_t iEnd = mode == -1 ? merger.Memory()->nRetryRefit : merger.NMergedTracks();
24+
constexpr bool rebuilt = I == GPUTPCGMMergerTrackFit::rebuild, retry = I == GPUTPCGMMergerTrackFit::retry;
25+
const int32_t iEnd = retry ? merger.Memory()->nRetryRefit : merger.NMergedTracks();
2526
GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, iEnd, {
26-
const int32_t i = mode == -1 ? merger.RetryRefitIds()[ii] : (mode ? merger.TrackOrderProcess()[ii] : ii);
27-
GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt, mode == -1);
27+
int32_t i;
28+
if constexpr (retry) {
29+
i = merger.RetryRefitIds()[ii];
30+
} else if constexpr (GPUCA_PAR_SORT_BEFORE_FIT > 0) {
31+
i = merger.TrackOrderProcess()[ii];
32+
} else {
33+
i = ii;
34+
}
35+
(GPUTPCGMTrackParam::RefitTrack<rebuilt, retry>(merger.MergedTracks()[i], i, merger)); // Parentheses, since the comma of the template arguments would split the macro arguments
2836
});
2937
}
3038

39+
#ifndef GPUCA_GPUCODE // The host needs explicit instantiations, the kernels are called from another compilation unit
40+
template void GPUTPCGMMergerTrackFit::Thread<GPUTPCGMMergerTrackFit::def>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger);
41+
template void GPUTPCGMMergerTrackFit::Thread<GPUTPCGMMergerTrackFit::rebuild>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger);
42+
template void GPUTPCGMMergerTrackFit::Thread<GPUTPCGMMergerTrackFit::retry>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger);
43+
#endif
44+
3145
template <>
3246
GPUdii() void GPUTPCGMMergerFollowLoopers::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger)
3347
{

‎GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,11 @@ class GPUTPCGMMergerGeneral : public GPUKernelTemplate
3535
class GPUTPCGMMergerTrackFit : public GPUTPCGMMergerGeneral
3636
{
3737
public:
38+
enum K { def = 0,
39+
rebuild = 1,
40+
retry = 2 };
3841
template <int32_t iKernel = defaultKernel>
39-
GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t mode, int32_t rebuilt);
42+
GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger);
4043
};
4144

4245
class GPUTPCGMMergerFollowLoopers : public GPUTPCGMMergerGeneral

‎GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,13 @@ static thread_local bool CADEBUG_CONDITION = true;
5656
using namespace o2::gpu;
5757
using namespace o2::tpc;
5858

59-
GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt, bool retryAttempt)
59+
template <bool rebuiltKernel, bool retryAttempt>
60+
GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track)
6061
{
6162
constexpr float maxSinPhi = constants::MAX_SIN_PHI;
6263

6364
const GPUParam& GPUrestrict() param = merger.Param();
65+
const bool rebuilt = retryAttempt ? param.rec.tpc.rebuildTrackInFit : rebuiltKernel; // The retry runs on the rebuilt tracks if the rebuild is enabled
6466
const float maxSinForUpdate = param.rec.tpc.maxSinPhiForFitUpdate;
6567
GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef();
6668

@@ -1488,19 +1490,21 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const
14881490
return ok;
14891491
}
14901492

1491-
GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt, bool retryAttempt)
1493+
template <bool rebuiltKernel, bool retryAttempt>
1494+
GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger)
14921495
{
14931496
if (!track.OK()) {
14941497
return;
14951498
}
1499+
const bool rebuilt = retryAttempt ? merger.Param().rec.tpc.rebuildTrackInFit : rebuiltKernel;
14961500

14971501
CADEBUG3(if (GPUCA_CADEBUG_ENABLED >= 2) { static const std::set<int32_t> dbgTracks = []() { std::set<int32_t> m; if (const char* e = getenv("DEBUG_TRACK")) { std::string s(e); for (char* t = strtok(s.data(), ";, \n\t"); t; t = strtok(nullptr, ";, \n\t")) { m.insert(atoi(t)); } } return m; }(); CADEBUG_CONDITION = dbgTracks.count(iTrk); });
14981502

14991503
int32_t nTrackHits = track.NClusters();
15001504
int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut
15011505
GPUTPCGMTrackParam t = track.Param();
15021506
float Alpha = track.Alpha();
1503-
bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt, retryAttempt);
1507+
bool ok = t.Fit<rebuiltKernel, retryAttempt>(merger, iTrk, nTrackHits, NTolerated, Alpha, track);
15041508
CADEBUG(if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits)));
15051509

15061510
if (!ok && (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) && !retryAttempt && merger.Param().rec.tpc.retryRefit) {
@@ -1536,6 +1540,12 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict()
15361540
// if (track.OK()) merger.DebugRefitMergedTrack(track);
15371541
}
15381542

1543+
#ifndef GPUCA_GPUCODE // The host needs explicit instantiations, RefitTrack is called from another compilation unit
1544+
template void GPUTPCGMTrackParam::RefitTrack<false, false>(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger);
1545+
template void GPUTPCGMTrackParam::RefitTrack<true, false>(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger);
1546+
template void GPUTPCGMTrackParam::RefitTrack<false, true>(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger);
1547+
#endif
1548+
15391549
GPUd() void GPUTPCGMTrackParam::Rotate(float alpha)
15401550
{
15411551
float cA, sA;

0 commit comments

Comments
 (0)