Skip to content

Commit 379e9b6

Browse files
committed
Mutex-free TGeo material budget in Propagator for multithreaded users
Propagator::getMatBudget (TGeo branch) used the shared TGeo navigator under GeometryManager::sTGMutex. Add opt-in per-thread navigators: * GeometryManager::setMaxThreads(n): wraps TGeoManager::SetMaxThreads and raises the flag enabling per-thread navigators (one-way, like TGeo's). * GeometryManager::getThreadNavigator(): lazily books a thread_local navigator; returns nullptr if setMaxThreads() was never called or if more than n threads ask for one, in which case meanMaterialBudget(..., nullptr) falls back to the old mutexed shared-navigator path. * Propagator::getMatBudget passes getThreadNavigator() to GeometryManager::meanMaterialBudget. * MatLayerCylSet LUT build uses GeometryManager::setMaxThreads. How to use in code calling Propagator methods from several threads: 1. Load the geometry (GeometryManager::loadGeometry or from CCDB). 2. Serially, BEFORE starting the worker threads, call o2::base::GeometryManager::setMaxThreads(nThreads); with nThreads >= the maximum number of threads that will call the Propagator (it walks all volumes, so do it once, not per call). 3. Call Propagator methods (propagateToDCA, propagateTo with MatCorrType::USEMatCorrTGeo, or with the TGeo fall-back when no MatLUT) from the worker threads as usual; each thread gets its own navigator on first use and releases it on thread exit. No other change needed. Without step 2 behaviour is unchanged (mutexed, thread-safe but serialized). Threads beyond nThreads log a warning and use the mutexed path. Using a MatLayerCylSet LUT (MatCorrType::USEMatCorrLUT) never touches TGeo. Co-Authored-By: Claude Sonnet 5.5
1 parent cf1343c commit 379e9b6

4 files changed

Lines changed: 65 additions & 3 deletions

File tree

‎Detectors/Base/include/DetectorsBase/GeometryManager.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include "GPUCommonLogger.h" // for LOG
2727
#include "MathUtils/Cartesian.h"
2828
#include "DetectorsBase/MatCell.h"
29+
#include <atomic>
2930
#include <mutex>
3031
#include <vector>
3132
class TGeoHMatrix; // lines 11-11
@@ -105,6 +106,17 @@ class GeometryManager : public TObject
105106
ClassDefNV(MatBudgetExt, 1);
106107
};
107108

109+
/// Enable multithreaded TGeo navigation for up to nThreads threads (wraps TGeoManager::SetMaxThreads,
110+
/// which is one-way and must be called serially, before the threads start navigating).
111+
/// Until this is called, getThreadNavigator() returns nullptr and callers fall back to the mutexed path.
112+
static void setMaxThreads(int nThreads);
113+
static int getMaxThreads() { return sMaxThreads.load(); }
114+
115+
/// Navigator owned by the calling thread, to be passed to meanMaterialBudget(..., nav).
116+
/// Returns nullptr (=> mutexed shared navigator) if setMaxThreads() was not called or if
117+
/// more than the declared number of threads asked for a navigator.
118+
static TGeoNavigator* getThreadNavigator();
119+
108120
/// Mean material budget between two points. Pass a navigator owned by the calling thread to
109121
/// run lock-free from several threads; with nav = nullptr the shared navigator is used under
110122
/// a mutex, as before.
@@ -182,6 +194,8 @@ class GeometryManager : public TObject
182194
static constexpr UInt_t sDetOffsetLarge = 17; /// detector identifier will start from this bit for detectors after the FOC
183195
static constexpr UInt_t sSensorMaskLarge = (0x1 << sDetOffsetLarge) - 1; /// mask=max sensitive volumes allowed per detector (131071)
184196
static std::mutex sTGMutex;
197+
static std::atomic<int> sMaxThreads; // 0: TGeo multithreading not enabled
198+
static std::atomic<int> sNNavigators; // per-thread navigators booked so far
185199

186200
ClassDefOverride(GeometryManager, 0); // Manager of geometry information for alignment
187201
};

‎Detectors/Base/src/GeometryManager.cxx‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,54 @@ using namespace o2::base;
6666
/// the look-up table mapping unique volume indices to symbolic volume names. For that, it
6767
/// collects several static methods
6868
std::mutex GeometryManager::sTGMutex;
69+
std::atomic<int> GeometryManager::sMaxThreads{0};
70+
std::atomic<int> GeometryManager::sNNavigators{0};
71+
72+
//_____________________________________________________________________________________
73+
void GeometryManager::setMaxThreads(int nThreads)
74+
{
75+
// must be called serially, before threads start using getThreadNavigator()
76+
if (!gGeoManager) {
77+
LOG(fatal) << "geometry is not loaded";
78+
}
79+
std::lock_guard<std::mutex> guard(sTGMutex);
80+
if (nThreads > sMaxThreads.load()) {
81+
gGeoManager->SetMaxThreads(nThreads); // one-way in TGeoManager
82+
sMaxThreads = nThreads;
83+
}
84+
}
85+
86+
//_____________________________________________________________________________________
87+
TGeoNavigator* GeometryManager::getThreadNavigator()
88+
{
89+
struct Holder {
90+
TGeoNavigator* nav = nullptr;
91+
bool tried = false;
92+
~Holder()
93+
{
94+
if (nav && gGeoManager) {
95+
std::lock_guard<std::mutex> guard(sTGMutex);
96+
gGeoManager->RemoveNavigator(nav);
97+
sNNavigators--;
98+
}
99+
}
100+
};
101+
thread_local Holder h;
102+
if (!h.nav && !h.tried) {
103+
if (sMaxThreads.load() == 0 || !gGeoManager) {
104+
return nullptr; // not enabled: do not latch, it may be enabled later
105+
}
106+
h.tried = true;
107+
std::lock_guard<std::mutex> guard(sTGMutex);
108+
if (sNNavigators.load() < sMaxThreads.load()) {
109+
h.nav = gGeoManager->AddNavigator();
110+
sNNavigators++;
111+
} else {
112+
LOG(warning) << "More threads than declared in GeometryManager::setMaxThreads(" << sMaxThreads.load() << "), using mutexed TGeo navigator";
113+
}
114+
}
115+
return h.nav;
116+
}
69117

70118
//______________________________________________________________________
71119
Bool_t GeometryManager::getOriginalMatrix(const char* symname, TGeoHMatrix& m)

‎Detectors/Base/src/MatLayerCylSet.cxx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ void MatLayerCylSet::populateFromTGeo(int ntrPerCell, int nThreads, MatbudGeomBa
179179
// single-threaded mode -- so we do not pretend to restore it; that is harmless because
180180
// meanMaterialBudget() decides whether to lock from its own argument, not from this global.
181181
// The navigators we book are ours, though, so those we do give back.
182-
gGeoManager->SetMaxThreads(nThreads);
182+
GeometryManager::setMaxThreads(nThreads);
183183

184184
tbb::enumerable_thread_specific<TGeoNavigator*> threadNavigators(
185185
[]() { return gGeoManager->AddNavigator(); });

‎Detectors/Base/src/Propagator.cxx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1011,11 +1011,11 @@ GPUd() MatBudget PropagatorImpl<value_T>::getMatBudget(PropagatorImpl<value_type
10111011
{
10121012
#if !defined(GPUCA_STANDALONE) && !defined(GPUCA_GPUCODE)
10131013
if (corrType == MatCorrType::USEMatCorrTGeo) {
1014-
return GeometryManager::meanMaterialBudget(p0, p1);
1014+
return GeometryManager::meanMaterialBudget(p0, p1, GeometryManager::getThreadNavigator());
10151015
}
10161016
if (!mMatLUT) {
10171017
if (mTGeoFallBackAllowed) {
1018-
return GeometryManager::meanMaterialBudget(p0, p1);
1018+
return GeometryManager::meanMaterialBudget(p0, p1, GeometryManager::getThreadNavigator());
10191019
} else {
10201020
throw std::runtime_error("requested MatLUT is absent and fall-back to TGeo is disabled");
10211021
}

0 commit comments

Comments
 (0)