Geant4-11
G4TaskRunManager.cc
Go to the documentation of this file.
1//
2// ********************************************************************
3// * License and Disclaimer *
4// * *
5// * The Geant4 software is copyright of the Copyright Holders of *
6// * the Geant4 Collaboration. It is provided under the terms and *
7// * conditions of the Geant4 Software License, included in the file *
8// * LICENSE and available at http://cern.ch/geant4/license . These *
9// * include a list of copyright holders. *
10// * *
11// * Neither the authors of this software system, nor their employing *
12// * institutes,nor the agencies providing financial support for this *
13// * work make any representation or warranty, express or implied, *
14// * regarding this software system or assume any liability for its *
15// * use. Please see the license in the file LICENSE and URL above *
16// * for the full disclaimer and the limitation of liability. *
17// * *
18// * This code implementation is the result of the scientific and *
19// * technical work of the GEANT4 collaboration. *
20// * By using, copying, modifying or distributing the software (or *
21// * any work based on the software) you agree to acknowledge its *
22// * use in resulting scientific publications, and indicate your *
23// * acceptance of all terms of the Geant4 Software license. *
24// ********************************************************************
25//
26//
27
28#include "G4TaskRunManager.hh"
29#include "G4AutoLock.hh"
30#include "G4EnvironmentUtils.hh"
32#include "G4Run.hh"
33#include "G4ScoringManager.hh"
34#include "G4StateManager.hh"
35#include "G4Task.hh"
36#include "G4TaskGroup.hh"
37#include "G4TaskManager.hh"
39#include "G4ThreadPool.hh"
40#include "G4Threading.hh"
41#include "G4TiMemory.hh"
42#include "G4Timer.hh"
44#include "G4UImanager.hh"
45#include "G4UserRunAction.hh"
50#include "G4WorkerThread.hh"
51#include "G4UserTaskQueue.hh"
52#include "G4TiMemory.hh"
54
55#include <cstdlib>
56#include <cstring>
57#include <iterator>
58
59//============================================================================//
60
61namespace
62{
66} // namespace
67
68//============================================================================//
69
71{
73}
74
75//============================================================================//
76
78 G4int grainsize)
80 , PTL::TaskRunManager(useTBB)
81 , eventGrainsize(grainsize)
82 , numberOfEventsPerTask(-1)
83 , numberOfTasks(-1)
84 , masterRNGEngine(nullptr)
85 , workTaskGroup(nullptr)
86{
87 if(task_queue)
88 taskQueue = task_queue;
89
90 // override default of 2 from G4MTRunManager
92 fMasterRM = this;
93 MTkernel = static_cast<G4TaskRunManagerKernel*>(kernel);
94
95 G4int numberOfStaticAllocators = kernel->GetNumberOfStaticAllocators();
96 if(numberOfStaticAllocators > 0)
97 {
99 msg1 << "There are " << numberOfStaticAllocators
100 << " static G4Allocator objects detected.\n"
101 << "In multi-threaded mode, all G4Allocator objects must "
102 << "be dynamicly instantiated.";
103 G4Exception("G4TaskRunManager::G4TaskRunManager", "Run1035", FatalException,
104 msg1);
105 }
106
109
110 // use default RandomNumberGenerator if created by user, or create default
111 masterRNGEngine = G4Random::getTheEngine();
112
115
116 //------------------------------------------------------------------------//
117 // handle threading
118 //------------------------------------------------------------------------//
119 G4String _nthread_env = G4GetEnv<G4String>("G4FORCENUMBEROFTHREADS", "");
120 for(auto& itr : _nthread_env)
121 itr = tolower(itr);
122
123 if(_nthread_env == "max")
125 else if(!_nthread_env.empty())
126 {
127 std::stringstream ss;
128 G4int _nthread_val = -1;
129 ss << _nthread_env;
130 ss >> _nthread_val;
131 if(_nthread_val > 0)
132 forcedNwokers = _nthread_val;
133
134 if(forcedNwokers > 0)
136 }
137
138 //------------------------------------------------------------------------//
139 // option for forcing TBB
140 //------------------------------------------------------------------------//
141#ifdef GEANT4_USE_TBB
142 G4int _useTBB = G4GetEnv<G4int>("G4FORCE_TBB", (G4int) useTBB);
143 if(_useTBB > 0)
144 useTBB = true;
145#else
146 if(useTBB)
147 {
149 msg << "TBB was requested but Geant4 was not built with TBB support";
150 G4Exception("G4TaskRunManager::G4TaskRunManager(...)", "Run0131",
151 JustWarning, msg);
152 }
153 useTBB = false;
154#endif
155
156 // handle TBB
158}
159
160//============================================================================//
161
163 : G4TaskRunManager(nullptr, useTBB, 0)
164{}
165
166//============================================================================//
167
169{
170 // finalize profiler before shutting down the threads
172
173 // terminate all the workers
175
176 // trigger all G4AutoDelete instances
178
179 // delete the task-group
180 delete workTaskGroup;
181 workTaskGroup = nullptr;
182
183 // destroy the thread-pool
184 if(threadPool)
186
188}
189
190//============================================================================//
191
193{
195}
196
197//============================================================================//
198
200{
201 std::ostringstream os;
202 os << randomNumberStatusDir << "G4Master_" << fn << ".rndm";
203 G4Random::saveEngineStatus(os.str().c_str());
204}
205
206//============================================================================//
207
209{
210 if(forcedNwokers > 0)
211 {
212 if(verboseLevel > 0)
213 {
215 msg << "\n### Number of threads is forced to " << forcedNwokers
216 << " by G4FORCENUMBEROFTHREADS environment variable. G4TaskRunManager::"
217 << __FUNCTION__ << "(" << n << ") ignored ###";
218 G4Exception("G4TaskRunManager::SetNumberOfThreads(G4int)", "Run0132",
219 JustWarning, msg);
220 }
222 }
223 else
224 {
225 nworkers = n;
227 {
228 if(verboseLevel > 0)
229 {
230 std::stringstream ss;
231 ss << "\n### Thread-pool already initialized. Resizing to " << nworkers
232 << "threads ###";
233 G4cout << ss.str() << "\n" << G4endl;
234 }
236 }
237 }
238}
239
240//============================================================================//
241
243{
244 G4bool firstTime = (!threadPool);
245 if(firstTime)
247
249
250 // make sure all worker threads are set up.
252 if(firstTime)
254 // G4UImanager::GetUIpointer()->SetIgnoreCmdNotFound(true);
255}
256
257//============================================================================//
258
260{
262 {
263 G4Exception("G4TaskRunManager::InitializeThreadPool", "Run1040",
264 JustWarning, "Threadpool already initialized. Ignoring...");
265 return;
266 }
267
270
271 // create the joiners
272 if(!workTaskGroup)
274
275 if(verboseLevel > 0)
276 {
277 std::stringstream ss;
278 ss.fill('=');
279 ss << std::setw(90) << "";
280 G4cout << "\n" << ss.str() << G4endl;
281
283 {
284 G4cout << "G4TaskRunManager :: Using TBB..." << G4endl;
285 }
286 else
287 {
288 G4cout << "G4TaskRunManager :: Using G4ThreadPool..." << G4endl;
289 }
290
291 G4cout << ss.str() << "\n" << G4endl;
292 }
293}
294
295//============================================================================//
296
298{
299 // Nothing to do
300}
301
302//============================================================================//
303
305{
306 // Nothing to do
307}
308
309//============================================================================//
310
312{
313 G4int grainSize = (eventGrainsize == 0) ? threadPool->size() : eventGrainsize;
314 grainSize =
315 G4GetEnv<G4int>("G4FORCE_GRAINSIZE", grainSize, "Forcing grainsize...");
316 if(grainSize == 0)
317 grainSize = 1;
318
319 G4int nEvtsPerTask = (numberOfEventToBeProcessed > grainSize)
320 ? (numberOfEventToBeProcessed / grainSize)
321 : 1;
322
323 if(eventModuloDef > 0)
324 {
326 }
327 else
328 {
330 if(eventModulo < 1)
331 eventModulo = 1;
332 }
333 if(eventModulo > nEvtsPerTask)
334 {
335 G4int oldMod = eventModulo;
336 eventModulo = nEvtsPerTask;
337
339 msgd << "Event modulo is reduced to " << eventModulo << " (was " << oldMod
340 << ")"
341 << " to distribute events to all threads.";
342 G4Exception("G4TaskRunManager::InitializeEventLoop()", "Run10035",
343 JustWarning, msgd);
344 }
345 nEvtsPerTask = eventModulo;
346
347 if(fakeRun)
348 nEvtsPerTask = G4GetEnv<G4int>(
349 "G4FORCE_EVENTS_PER_TASK", nEvtsPerTask,
350 "Forcing number of events per task (overrides grainsize)...");
351 else
352 nEvtsPerTask = G4GetEnv<G4int>("G4FORCE_EVENTS_PER_TASK", nEvtsPerTask);
353
354 if(nEvtsPerTask < 1)
355 nEvtsPerTask = 1;
356
358 numberOfEventsPerTask = nEvtsPerTask;
360
361 if(fakeRun && verboseLevel > 1)
362 {
363 std::stringstream msg;
364 msg << "--> G4TaskRunManager::ComputeNumberOfTasks() --> " << numberOfTasks
365 << " tasks with " << numberOfEventsPerTask << " events/task...";
366
367 std::stringstream ss;
368 ss.fill('=');
369 ss << std::setw(msg.str().length()) << "";
370 G4cout << "\n"
371 << ss.str() << "\n"
372 << msg.str() << "\n"
373 << ss.str() << "\n"
374 << G4endl;
375 }
376}
377
378//============================================================================//
379
381{
382 // Now loop on requested number of workers
383 // This will also start the workers
384 // Currently we do not allow to change the
385 // number of threads: threads area created once
386 // Instead of pthread based workers, create tbbTask
387 static bool initializeStarted = false;
388
390
391 if(fakeRun)
392 {
393 if(initializeStarted)
394 {
395 auto initCmdStack = GetCommandStack();
396 if(!initCmdStack.empty())
397 {
398 threadPool->execute_on_all_threads([initCmdStack]() {
399 for(auto& itr : initCmdStack)
400 G4UImanager::GetUIpointer()->ApplyCommand(itr);
402 });
403 }
404 }
405 else
406 {
407 std::stringstream msg;
408 msg << "--> G4TaskRunManager::CreateAndStartWorkers() --> "
409 << "Initializing workers...";
410
411 std::stringstream ss;
412 ss.fill('=');
413 ss << std::setw(msg.str().length()) << "";
414 G4cout << "\n"
415 << ss.str() << "\n"
416 << msg.str() << "\n"
417 << ss.str() << "\n"
418 << G4endl;
419
423 }
424 initializeStarted = true;
425 }
426 else
427 {
428 auto initCmdStack = GetCommandStack();
429 if(!initCmdStack.empty())
430 {
431 threadPool->execute_on_all_threads([initCmdStack]() {
432 for(auto& itr : initCmdStack)
433 G4UImanager::GetUIpointer()->ApplyCommand(itr);
434 });
435 }
436
437 // cleans up a previous run and events in case a thread
438 // does not execute any tasks
441
442 {
443 std::stringstream msg;
444 msg << "--> G4TaskRunManager::CreateAndStartWorkers() --> "
445 << "Creating " << numberOfTasks << " tasks with "
446 << numberOfEventsPerTask << " events/task...";
447
448 std::stringstream ss;
449 ss.fill('=');
450 ss << std::setw(msg.str().length()) << "";
451 G4cout << "\n"
452 << ss.str() << "\n"
453 << msg.str() << "\n"
454 << ss.str() << "\n"
455 << G4endl;
456 }
457
459 for(G4int nt = 0; nt < numberOfTasks + 1; ++nt)
460 {
461 if(remaining > 0)
462 AddEventTask(nt);
463 remaining -= numberOfEventsPerTask;
464 }
466 }
467}
468
469//============================================================================//
470
472{
473 if(verboseLevel > 1)
474 G4cout << "Adding task " << nt << " to task-group..." << G4endl;
476}
477
478//============================================================================//
479
481{
483 G4int nFill = 0;
485 {
486 case 0:
488 break;
489 case 1:
490 nFill = numberOfTasks - nSeedsFilled;
491 break;
492 case 2:
493 default:
496 1;
497 }
498 // Generates up to nSeedsMax seed pairs only.
499 if(nFill > nSeedsMax)
500 nFill = nSeedsMax;
502 helper->Refill(randDbl, nFill);
503 nSeedsFilled += nFill;
504}
505
506//============================================================================//
507
508void G4TaskRunManager::InitializeEventLoop(G4int n_event, const char* macroFile,
509 G4int n_select)
510{
514
515 if(!fakeRun)
516 {
517 nSeedsUsed = 0;
518 nSeedsFilled = 0;
519
520 if(verboseLevel > 0)
521 timer->Start();
522
523 n_select_msg = n_select;
524 if(macroFile != nullptr)
525 {
526 if(n_select_msg < 0)
527 n_select_msg = n_event;
528
529 msgText = "/control/execute ";
530 msgText += macroFile;
531 selectMacro = macroFile;
532 }
533 else
534 {
535 n_select_msg = -1;
536 selectMacro = "";
537 }
538
540
541 // initialize seeds
542 // If user did not implement InitializeSeeds,
543 // use default: nSeedsPerEvent seeds per event
544
545 if(n_event > 0)
546 {
547 G4bool _overload = InitializeSeeds(n_event);
548 G4bool _functor = false;
549 if(!_overload)
550 _functor = initSeedsCallback(n_event, nSeedsPerEvent, nSeedsFilled);
551 if(_overload == false && _functor == false)
552 {
555 {
556 case 0:
557 nSeedsFilled = n_event;
558 break;
559 case 1:
561 break;
562 case 2:
563 nSeedsFilled = n_event / eventModulo + 1;
564 break;
565 default:
567 msgd << "Parameter value <" << SeedOncePerCommunication()
568 << "> of seedOncePerCommunication is invalid. It is reset "
569 "to 0.";
570 G4Exception("G4TaskRunManager::InitializeEventLoop()", "Run10036",
571 JustWarning, msgd);
573 nSeedsFilled = n_event;
574 }
575
576 // Generates up to nSeedsMax seed pairs only.
580 helper->Fill(randDbl, nSeedsFilled, n_event, nSeedsPerEvent);
581 }
582 }
583 }
584
585 // Now initialize workers. Check if user defined a WorkerThreadInitialization
586 if(userWorkerThreadInitialization == nullptr)
588
589 // Prepare UI commands for threads
591
592 // Start worker threads
594}
595
596//============================================================================//
597
599{
600 // Wait for all worker threads to have finished the run
601 // i.e. wait for them to return from RunTermination()
602 // This guarantee that userrunaction for workers has been called
603
604 // Wait now for all threads to finish event-loop
606 // Now call base-class methof
609}
610
611//============================================================================//
612
614{
616 // Call base class stuff...
618
619 masterWorlds.clear();
620 size_t nWorlds =
622 std::vector<G4VPhysicalVolume*>::iterator itrW =
624 for(size_t iWorld = 0; iWorld < nWorlds; ++iWorld)
625 {
626 addWorld(iWorld, *itrW);
627 ++itrW;
628 }
629}
630
631//============================================================================//
632
633void G4TaskRunManager::MergeScores(const G4ScoringManager* localScoringManager)
634{
636 if(masterScM)
637 masterScM->Merge(localScoringManager);
638}
639
640//============================================================================//
641
643{
645 if(currentRun)
646 currentRun->Merge(localRun);
647}
648
649//============================================================================//
650
652 G4long& s3, G4bool reseedRequired)
653{
656 {
658 if(reseedRequired)
659 {
661 G4int idx_rndm = nSeedsPerEvent * nSeedsUsed;
662 s1 = helper->GetSeed(idx_rndm);
663 s2 = helper->GetSeed(idx_rndm + 1);
664 if(nSeedsPerEvent == 3)
665 s3 = helper->GetSeed(idx_rndm + 2);
666 ++nSeedsUsed;
668 RefillSeeds();
669 }
671 return true;
672 }
673 return false;
674}
675
676//============================================================================//
677
679 G4bool reseedRequired)
680{
683 {
685 G4int nmod = eventModulo;
687 {
690 }
692
693 if(reseedRequired)
694 {
696 G4int nevRnd = nmod;
698 nevRnd = 1;
699 for(G4int i = 0; i < nevRnd; ++i)
700 {
701 seedsQueue->push(helper->GetSeed(nSeedsPerEvent * nSeedsUsed));
702 seedsQueue->push(helper->GetSeed(nSeedsPerEvent * nSeedsUsed + 1));
703 if(nSeedsPerEvent == 3)
704 seedsQueue->push(helper->GetSeed(nSeedsPerEvent * nSeedsUsed + 2));
705 nSeedsUsed++;
707 RefillSeeds();
708 }
709 }
711 return nevt;
712 }
713 return 0;
714}
715
716//============================================================================//
717
719{
720 // Force workers to execute (if any) all UI commands left in the stack
722
723 if(workTaskGroup)
724 {
726 if(!fakeRun)
729 }
730}
731
732//============================================================================//
733
735{
736 // This method is valid only for GeomClosed or EventProc state
737 G4ApplicationState currentState =
739 if(currentState == G4State_GeomClosed || currentState == G4State_EventProc)
740 {
741 runAborted = true;
742 MTkernel->BroadcastAbortRun(softAbort);
743 }
744 else
745 {
746 G4cerr << "Run is not in progress. AbortRun() ignored." << G4endl;
747 }
748}
749
750//============================================================================//
751
753{
754 // nothing to do in the master thread
755}
756
757//============================================================================//
758
760{
761 if(workTaskGroup)
762 {
764 if(!fakeRun)
767 }
768}
769
770//============================================================================//
771
773{
775
776 auto process_commands_stack = []() {
778 if(mrm)
779 {
780 auto cmds = mrm->GetCommandStack();
781 for(const auto& itr : cmds)
782 G4UImanager::GetUIpointer()->ApplyCommand(itr); // TLS instance
784 }
785 };
786
787 if(threadPool)
788 threadPool->execute_on_all_threads(process_commands_stack);
789}
790
791//============================================================================//
792
794
795//============================================================================//
G4ApplicationState
@ G4State_EventProc
@ G4State_GeomClosed
@ JustWarning
@ FatalException
void G4Exception(const char *originOfException, const char *exceptionCode, G4ExceptionSeverity severity, const char *description)
Definition: G4Exception.cc:35
std::ostringstream G4ExceptionDescription
Definition: G4Exception.hh:40
std::queue< G4long > G4SeedsQueue
Definition: G4RNGHelper.hh:136
G4Thread::id G4ThreadId
Definition: G4Threading.hh:286
std::mutex G4Mutex
Definition: G4Threading.hh:81
double G4double
Definition: G4Types.hh:83
long G4long
Definition: G4Types.hh:87
bool G4bool
Definition: G4Types.hh:86
int G4int
Definition: G4Types.hh:85
G4GLOB_DLL std::ostream G4cerr
#define G4endl
Definition: G4ios.hh:57
G4GLOB_DLL std::ostream G4cout
virtual void flatArray(const int size, double *vect)=0
void SetEventID(G4int i)
Definition: G4Event.hh:80
static G4int SeedOncePerCommunication()
static void SetSeedOncePerCommunication(G4int val)
G4int numberOfEventToBeProcessed
virtual void ThisWorkerProcessCommandsStackDone()
static void addWorld(G4int counter, G4VPhysicalVolume *w)
virtual void PrepareCommandsStack()
static G4MTRUN_DLL G4ScoringManager * masterScM
static G4MTRUN_DLL G4MTRunManager * fMasterRM
G4double * randDbl
static G4MTRUN_DLL masterWorlds_t masterWorlds
static G4MTRunManager * GetMasterRunManager()
static G4ThreadId GetMasterThreadId()
std::vector< G4String > GetCommandStack()
static void Finalize()
Definition: G4Profiler.cc:331
G4int GetNumberOfStaticAllocators() const
virtual void Initialize()
G4Timer * timer
void SetRunIDCounter(G4int i)
G4UserWorkerThreadInitialization * userWorkerThreadInitialization
G4int n_select_msg
G4int numberOfEventProcessed
G4RunManagerKernel * kernel
G4int verboseLevel
G4Run * currentRun
G4bool runAborted
G4String msgText
virtual void BeamOn(G4int n_event, const char *macroFile=nullptr, G4int n_select=-1)
G4String selectMacro
virtual void RunTermination()
G4String randomNumberStatusDir
virtual void TerminateEventLoop()
virtual void ConstructScoringWorlds()
Definition: G4Run.hh:49
virtual void Merge(const G4Run *)
Definition: G4Run.cc:67
void Merge(const G4ScoringManager *scMan)
static G4ScoringManager * GetScoringManagerIfExist()
const G4ApplicationState & GetCurrentState() const
static G4StateManager * GetStateManager()
void BroadcastAbortRun(G4bool softAbort)
static std::vector< G4String > & InitCommandStack()
virtual void ConstructScoringWorlds() override
virtual void InitializeThreadPool() override
virtual void CreateAndStartWorkers() override
static G4TaskRunManager * GetMasterRunManager()
virtual G4bool InitializeSeeds(G4int) override
virtual void ThisWorkerProcessCommandsStackDone() override
virtual G4int SetUpNEvents(G4Event *, G4SeedsQueue *seedsQueue, G4bool reseedRequired=true) override
G4VUserTaskQueue *& taskQueue
virtual void SetNumberOfThreads(G4int n) override
virtual void AbortRun(G4bool softAbort=false) override
InitializeSeedsCallback initSeedsCallback
virtual void RefillSeeds() override
virtual void RequestWorkersProcessCommandsStack() override
virtual void AddEventTask(G4int)
static G4ThreadId GetMasterThreadId()
virtual void TerminateOneEvent() override
RunTaskGroup * workTaskGroup
CLHEP::HepRandomEngine * masterRNGEngine
virtual void TerminateWorkers() override
virtual void AbortEvent() override
virtual void InitializeEventLoop(G4int n_event, const char *macroFile=nullptr, G4int n_select=-1) override
virtual ~G4TaskRunManager()
virtual void Initialize() override
virtual void StoreRNGStatus(const G4String &filenamePrefix) override
G4TaskRunManagerKernel * MTkernel
virtual void RunTermination() override
void MergeScores(const G4ScoringManager *localScoringManager)
virtual void WaitForEndEventLoopWorkers() override
G4TaskGroup< void > RunTaskGroup
G4ThreadPool *& threadPool
static G4TaskRunManagerKernel * GetMTMasterRunManagerKernel()
virtual void ProcessOneEvent(G4int i_event) override
void MergeRun(const G4Run *localRun)
virtual void ComputeNumberOfTasks()
G4TaskRunManager(G4bool useTBB=G4GetEnv< G4bool >("G4USE_TBB", false))
virtual G4bool SetUpAnEvent(G4Event *, G4long &s1, G4long &s2, G4long &s3, G4bool reseedRequired=true) override
static G4TemplateRNGHelper< T > * GetInstance()
Definition: G4RNGHelper.cc:42
virtual const T GetSeed(const G4int &sdId)
Definition: G4RNGHelper.hh:64
void Fill(G4double *dbl, G4int nev, G4int nev_tot, G4int nrpe)
Definition: G4RNGHelper.hh:83
void Refill(G4double *dbl, G4int nev)
Definition: G4RNGHelper.hh:96
void Start()
static G4TransportationManager * GetTransportationManager()
std::vector< G4VPhysicalVolume * >::iterator GetWorldsIterator()
size_t GetNoWorlds() const
G4int ApplyCommand(const char *aCommand)
Definition: G4UImanager.cc:485
void SetMasterUIManager(G4bool val)
Definition: G4UImanager.hh:219
static G4UImanager * GetUIpointer()
Definition: G4UImanager.cc:77
virtual void DoWork() override
static G4WorkerTaskRunManager * GetWorkerRunManager()
Up join(Up accum={})
enable_if_t< std::is_void< Up >::value, void > exec(Func func, Args... args)
int GetVerbose() const
virtual void Initialize(uint64_t n=std::thread::hardware_concurrency())
virtual void Terminate()
void SetVerbose(int val)
ThreadPool * GetThreadPool() const
void resize(size_type _n)
Definition: ThreadPool.hh:342
static void set_use_tbb(bool val)
Definition: ThreadPool.cc:102
bool is_tbb_threadpool() const
Definition: ThreadPool.hh:130
void execute_on_all_threads(FuncT &&_func)
Definition: ThreadPool.hh:423
size_type size() const
Definition: ThreadPool.hh:163
size_type destroy_threadpool()
Definition: ThreadPool.cc:364
G4int G4GetNumberOfCores()
Definition: G4Threading.cc:121
Definition: AutoLock.hh:254