A service which initializes a TBB thread pool. More...

#include </builds/gaudi/Gaudi/GaudiHive/src/ThreadPoolSvc.h>

Inheritance diagram for ThreadPoolSvc:

[legend]

Collaboration diagram for ThreadPoolSvc:

[legend]

Public Member Functions
	ThreadPoolSvc (const std::string &name, ISvcLocator *svc)
	Constructor. More...

StatusCode	initialize () override final
	Initialise. More...

StatusCode	finalize () override final
	Finalise. More...

StatusCode	initPool (const int &poolSize, const int &maxParallelismExtra) override final
	Initialize the thread pool and launch the ThreadInitTasks. More...

StatusCode	terminatePool () override final
	Terminate the thread pool and launch thread termination tasks. More...

int	poolSize () const override final

virtual bool	isInit () const

virtual void	initThisThread () override

tbb::task_arena *	getArena ()

Public Member Functions inherited from extends< Service, IThreadPoolSvc >
void *	i_cast (const InterfaceID &tid) const override
	Implementation of IInterface::i_cast. More...

StatusCode	queryInterface (const InterfaceID &ti, void **pp) override
	Implementation of IInterface::queryInterface. More...

std::vector< std::string >	getInterfaceNames () const override
	Implementation of IInterface::getInterfaceNames. More...

Public Member Functions inherited from Service
const std::string &	name () const override
	Retrieve name of the service More...

StatusCode	configure () override

StatusCode	initialize () override

StatusCode	start () override

StatusCode	stop () override

StatusCode	finalize () override

StatusCode	terminate () override

Gaudi::StateMachine::State	FSMState () const override

Gaudi::StateMachine::State	targetFSMState () const override

StatusCode	reinitialize () override

StatusCode	restart () override

StatusCode	sysInitialize () override
	Initialize Service More...

StatusCode	sysStart () override
	Initialize Service More...

StatusCode	sysStop () override
	Initialize Service More...

StatusCode	sysFinalize () override
	Finalize Service More...

StatusCode	sysReinitialize () override
	Re-initialize the Service. More...

StatusCode	sysRestart () override
	Re-initialize the Service. More...

	Service (std::string name, ISvcLocator *svcloc)
	Standard Constructor More...

SmartIF< ISvcLocator > &	serviceLocator () const override
	Retrieve pointer to service locator More...

template<class T >
StatusCode	service (const std::string &name, const T *&psvc, bool createIf=true) const
	Access a service by name, creating it if it doesn't already exist. More...

template<class T >
StatusCode	service (const std::string &name, T *&psvc, bool createIf=true) const

template<typename IFace = IService>
SmartIF< IFace >	service (const std::string &name, bool createIf=true) const

template<class T >
StatusCode	service (const std::string &svcType, const std::string &svcName, T *&psvc) const
	Access a service by name and type, creating it if it doesn't already exist. More...

template<class T >
Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, ToolHandle< T > &hndl, const std::string &doc="none")

template<class T >
StatusCode	declareTool (ToolHandle< T > &handle, bool createIf=true)

template<class T >
StatusCode	declareTool (ToolHandle< T > &handle, std::string toolTypeAndName, bool createIf=true)
	Declare used tool. More...

template<class T >
Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, ToolHandleArray< T > &hndlArr, const std::string &doc="none")

template<class T >
void	addToolsArray (ToolHandleArray< T > &hndlArr)

const std::vector< IAlgTool * > &	tools () const

SmartIF< IAuditorSvc > &	auditorSvc () const
	The standard auditor service.May not be invoked before sysInitialize() has been invoked. More...

Public Member Functions inherited from PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
	PropertyHolder ()=default

Gaudi::Details::PropertyBase &	declareProperty (Gaudi::Details::PropertyBase &prop)
	Declare a property. More...

Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, TYPE &value, const std::string &doc="none")
	Helper to wrap a regular data member and use it as a regular property. More...

Gaudi::Details::PropertyBase *	declareProperty (const std::string &name, Gaudi::Property< TYPE, VERIFIER, HANDLERS > &prop, const std::string &doc="none")
	Declare a PropertyBase instance setting name and documentation. More...

Gaudi::Details::PropertyBase *	declareRemoteProperty (const std::string &name, IProperty *rsvc, const std::string &rname="")
	Declare a remote property. More...

StatusCode	setProperty (const std::string &name, const Gaudi::Details::PropertyBase &p) override
	set the property from another property with a different name More...

StatusCode	setProperty (const std::string &s) override
	set the property from the formatted string More...

StatusCode	setProperty (const Gaudi::Details::PropertyBase &p)
	Set the property from a property. More...

virtual StatusCode	setProperty (const std::string &name, const Gaudi::Details::PropertyBase &p)=0
	Set the property from a property with a different name. More...

virtual StatusCode	setProperty (const std::string &s)=0
	Set the property by string. More...

StatusCode	setProperty (const std::string &name, const char *v)
	Special case for string literals. More...

StatusCode	setProperty (const std::string &name, const std::string &v)
	Special case for std::string. More...

StatusCode	setProperty (const std::string &name, const TYPE &value)
	set the property form the value More...

StatusCode	setPropertyRepr (const std::string &n, const std::string &r) override
	set the property from name and value string representation More...

StatusCode	getProperty (Gaudi::Details::PropertyBase *p) const override
	get the property More...

const Gaudi::Details::PropertyBase &	getProperty (std::string_view name) const override
	get the property by name More...

StatusCode	getProperty (std::string_view n, std::string &v) const override
	convert the property to the string More...

const std::vector< Gaudi::Details::PropertyBase * > &	getProperties () const override
	get all properties More...

bool	hasProperty (std::string_view name) const override
	Return true if we have a property with the given name. More...

Gaudi::Details::PropertyBase *	property (std::string_view name) const
	\fixme property and bindPropertiesTo should be protected More...

void	bindPropertiesTo (Gaudi::Interfaces::IOptionsSvc &optsSvc)

	PropertyHolder (const PropertyHolder &)=delete

PropertyHolder &	operator= (const PropertyHolder &)=delete

Public Member Functions inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
MSG::Level	msgLevel () const
	get the cached level (originally extracted from the embedded MsgStream) More...

bool	msgLevel (MSG::Level lvl) const
	get the output level from the embedded MsgStream More...

Private Member Functions
StatusCode	launchTasks (bool finalize=false)
	Launch tasks to execute the ThreadInitTools. More...

Private Attributes
ToolHandleArray< IThreadInitTool >	m_threadInitTools = { this }
	Handle array of thread init tools. More...

bool	m_init = false
	Was the thread pool initialized? More...

int	m_threadPoolSize = 0
	Size of the thread pool allocated. More...

tbb::spin_mutex	m_initMutex
	Mutex used to protect the initPool and terminatePool methods. More...

std::unique_ptr< boost::barrier >	m_barrier
	Barrier used to synchronization thread init tasks. More...

std::unique_ptr< tbb::global_control >	m_tbbgc
	TBB global control parameter. More...

tbb::task_arena	m_arena
	TBB task arena to run all algorithms. More...

std::atomic< int >	m_threadInitCount = 0
	Counter for all threads that are initialised. More...

Additional Inherited Members
Public Types inherited from extends< Service, IThreadPoolSvc >
using	base_class = extends
	Typedef to this class. More...

using	extend_interfaces_base = extend_interfaces< Interfaces... >
	Typedef to the base of this class. More...

Public Types inherited from Service
using	Factory = Gaudi::PluginService::Factory< IService (const std::string &, ISvcLocator )>

Public Types inherited from PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
using	PropertyHolderImpl = PropertyHolder< CommonMessaging< implements< IService, IProperty, IStateful > > >
	Typedef used to refer to this class from derived classes, as in. More...

Public Types inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
using	base_class = CommonMessaging

Public Types inherited from extend_interfaces< Interfaces... >
using	ext_iids = typename Gaudi::interface_list_cat< typename Interfaces::ext_iids... >::type
	take union of the ext_iids of all Interfaces... More...

Protected Member Functions inherited from Service
std::vector< IAlgTool * > &	tools ()

	~Service () override
	Standard Destructor More...

int	outputLevel () const
	get the `Service's` output level More...

Protected Member Functions inherited from CommonMessaging< implements< IService, IProperty, IStateful > >
MSG::Level	setUpMessaging () const
	Set up local caches. More...

MSG::Level	resetMessaging ()
	Reinitialize internal states. More...

void	updateMsgStreamOutputLevel (int level)
	Update the output level of the cached MsgStream. More...

Protected Attributes inherited from Service
Gaudi::StateMachine::State	m_state = Gaudi::StateMachine::OFFLINE
	Service state More...

Gaudi::StateMachine::State	m_targetState = Gaudi::StateMachine::OFFLINE
	Service state More...

Gaudi::Property< int >	m_outputLevel { this, "OutputLevel", MSG::NIL, "output level" }
	flag indicating whether ToolHandle tools have been added to m_tools More...

Gaudi::Property< bool >	m_auditInit { this, "AuditServices", false, "[[deprecated]] unused" }

Gaudi::Property< bool >	m_auditorInitialize { this, "AuditInitialize", false, "trigger auditor on initialize()" }

Gaudi::Property< bool >	m_auditorStart { this, "AuditStart", false, "trigger auditor on start()" }

Gaudi::Property< bool >	m_auditorStop { this, "AuditStop", false, "trigger auditor on stop()" }

Gaudi::Property< bool >	m_auditorFinalize { this, "AuditFinalize", false, "trigger auditor on finalize()" }

Gaudi::Property< bool >	m_auditorReinitialize { this, "AuditReinitialize", false, "trigger auditor on reinitialize()" }

Gaudi::Property< bool >	m_auditorRestart { this, "AuditRestart", false, "trigger auditor on restart()" }

Gaudi::Property< bool >	m_autoRetrieveTools

Gaudi::Property< bool >	m_checkToolDeps

SmartIF< IAuditorSvc >	m_pAuditorSvc
	Auditor Service More...

Detailed Description

A service which initializes a TBB thread pool.

This service can be configured with an array of IThreadInitTools which will each be invoked concurrently on each worker thread. A ThreadInitTask is created for each thread and given the list of tools. A boost::barrier is used to synchronize the calling of each tool concurrently on all threads at the same time.

Definition at line 38 of file ThreadPoolSvc.h.

Constructor & Destructor Documentation

◆ ThreadPoolSvc()

ThreadPoolSvc::ThreadPoolSvc	(	const std::string &	name,
		ISvcLocator *	svc
	)

Constructor.

Definition at line 32 of file ThreadPoolSvc.cpp.

                                                                          : extends( name, svcLoc ) {
   declareProperty( "ThreadInitTools", m_threadInitTools, "ToolHandleArray of IThreadInitTools" );
 }

Member Function Documentation

◆ finalize()

StatusCode ThreadPoolSvc::finalize ( )

finaloverride

Finalise.

Definition at line 62 of file ThreadPoolSvc.cpp.

                                    {
  
   if ( !m_init )
     warning() << "Looks like the ThreadPoolSvc was created, but thread pool "
               << "was never initialized" << endmsg;
  
   return StatusCode::SUCCESS;
 }

◆ getArena()

tbb::task_arena* ThreadPoolSvc::getArena ( )

inline

Definition at line 61 of file ThreadPoolSvc.h.

61 { return &m_arena; }

◆ initialize()

StatusCode ThreadPoolSvc::initialize ( )

finaloverride

Initialise.

Definition at line 38 of file ThreadPoolSvc.cpp.

                                      {
  
   // Initialise mother class (read properties, ...)
   if ( Service::initialize().isFailure() ) {
     warning() << "Base class could not be initialized" << endmsg;
     return StatusCode::FAILURE;
   }
  
   if ( m_threadInitTools.retrieve().isFailure() ) {
     error() << "Unable to retrieve ThreadInitTools Array" << endmsg;
  
     return StatusCode::FAILURE;
   }
  
   if ( m_threadInitTools.size() != 0 )
     info() << "retrieved " << m_threadInitTools.size() << " thread init tools" << endmsg;
   else
     info() << "no thread init tools attached" << endmsg;
  
   return StatusCode::SUCCESS;
 }

◆ initPool()

StatusCode ThreadPoolSvc::initPool	(	const int &	poolSize,
		const int &	maxParallelismExtra
	)

finaloverride

Initialize the thread pool and launch the ThreadInitTasks.

Definition at line 73 of file ThreadPoolSvc.cpp.

                                                                                         {
  
   tbb::spin_mutex::scoped_lock lock( m_initMutex );
  
   m_threadPoolSize = poolSize;
  
   ON_DEBUG debug() << "ThreadPoolSvc::initPool() poolSize = " << poolSize << endmsg;
   // There is a problem in the piece of the code below.  if
   // m_threadPoolSize is set to something negative which is < -1,
   // algorithm below might not behave as expected. For the time being
   // I've choosen to create the barrier with the default number of
   // threads created by the task scheduler init assuming that a
   // negative value will choose automatic thread creation which will
   // create default number of threads.
   // SK
  
   // -100 prevents the creation of the pool and the scheduler directly
   // executes the tasks.
   // -1 means use all available hardware threads
  
   if ( -100 != m_threadPoolSize ) {
     ON_DEBUG debug() << "Initialising a thread pool of size " << m_threadPoolSize << endmsg;
  
     if ( m_threadPoolSize == -1 ) {
       m_threadPoolSize = std::thread::hardware_concurrency();
     } else if ( m_threadPoolSize < -1 ) {
       fatal() << "Unexpected ThreadPoolSize \"" << m_threadPoolSize << "\". Allowed negative values are "
               << "-1 (use all available cores) and -100 (don't use a thread pool)" << endmsg;
       return StatusCode::FAILURE;
     }
  
     ON_VERBOSE verbose() << "Maximum allowed parallelism before adjusting: "
                          << tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) << endmsg;
  
     // to get the number of threads we need, request one thread more to account for how TBB calculates
     // its soft limit of the number of threads for the global thread pool
     m_tbbgc = std::make_unique<tbb::global_control>( tbb::global_control::max_allowed_parallelism,
                                                      m_threadPoolSize + maxParallelismExtra + 1 );
  
     Gaudi::Concurrency::ConcurrencyFlags::setNumThreads( m_threadPoolSize );
  
     // Create the task arena to run all algorithms
     m_arena.initialize( m_threadPoolSize + 1 );
  
     // Create the barrier for task synchronization at termination
     // (here we increase the number of threads by one to account for calling thread)
     m_barrier = std::make_unique<boost::barrier>( m_threadPoolSize + 1 );
  
   } else {
     // don't use a thread pool
     Gaudi::Concurrency::ConcurrencyFlags::setNumThreads( 1 );
     m_tbbgc = std::make_unique<tbb::global_control>( tbb::global_control::max_allowed_parallelism, 0 );
   }
  
   ON_DEBUG debug() << "Thread Pool initialization complete. Maximum allowed parallelism: "
                    << tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) << endmsg;
  
   m_init = true;
  
   return StatusCode::SUCCESS;
 }

◆ initThisThread()

void ThreadPoolSvc::initThisThread ( )

overridevirtual

Definition at line 234 of file ThreadPoolSvc.cpp.

                                    {
  
   if ( Gaudi::Concurrency::ThreadInitDone ) {
     // this should never happen
     error() << "initThisThread triggered, but thread already initialized" << endmsg;
     throw GaudiException( "initThisThread triggered, but thread already initialized", name(), StatusCode::FAILURE );
   }
  
   m_threadInitCount++;
   boost::barrier* noBarrier = nullptr;
   ThreadInitTask( m_threadInitTools, noBarrier, serviceLocator(), false )();
 }

◆ isInit()

virtual bool ThreadPoolSvc::isInit ( ) const

inlinevirtual

Definition at line 57 of file ThreadPoolSvc.h.

57 { return m_init; }

◆ launchTasks()

StatusCode ThreadPoolSvc::launchTasks ( bool finalize = false )

private

Launch tasks to execute the ThreadInitTools.

Definition at line 158 of file ThreadPoolSvc.cpp.

                                                       {
  
   const std::string taskType = terminate ? "termination" : "initialization";
  
   // If we have a thread pool (via a scheduler), then we want to queue
   // the tasks in TBB to execute on each thread.
   if ( tbb::global_control::active_value( tbb::global_control::max_allowed_parallelism ) > 0 ) {
  
     // Make a warning message if not all threads can be finalised
     if ( terminate and m_threadInitCount.load() != m_threadPoolSize and not m_threadInitTools.empty() ) {
       warning() << "Finalising " << m_threadPoolSize << " threads, but " << m_threadInitCount.load()
                 << " threads were initialised" << endmsg;
     }
  
     // Create one task for each worker thread in the pool
     for ( int i = 0; i < m_threadPoolSize; ++i ) {
       ON_DEBUG debug() << "creating ThreadInitTask " << i << endmsg;
  
       // Queue the task
       if ( !terminate ) m_threadInitCount++;
       m_arena.enqueue( ThreadInitTask( m_threadInitTools, m_barrier.get(), serviceLocator(), terminate ) );
       std::this_thread::sleep_for( std::chrono::milliseconds( 20 ) );
     }
  
     // Now wait for all the workers to reach the barrier
     ON_DEBUG debug() << "waiting at barrier for all ThreadInitTool to finish executing" << endmsg;
     m_barrier->wait();
  
     // Check to make sure all Tools were invoked.
     // I'm not sure this mechanism is worthwhile.
     for ( auto& t : m_threadInitTools ) {
       // Number of threads initialized but not terminated.
       int numInit = t->nInit();
       // Expected number based on the type of task.
       int expectedNumInit = terminate ? 0 : m_threadPoolSize;
       if ( numInit != expectedNumInit ) {
         std::ostringstream ost;
         ost << "not all threads " << ( terminate ? "terminated" : "initialized" ) << " for tool " << t << " : "
             << t->nInit() << " out of " << m_threadPoolSize << " are currently active.";
         if ( terminate ) {
           // it is likely the case that TBB activated new threads
           // late in the game, and extra initializations were done
           info() << ost.str() << endmsg;
         } else {
           error() << ost.str() << endmsg;
           return StatusCode::FAILURE;
         }
       }
     }
  
   }
  
   // In single-threaded mode, there is no scheduler, so we simply call
   // the task wrapper directly in this thread.
   else {
     ON_DEBUG debug() << "launching ThreadInitTask " << taskType << "in this thread." << endmsg;
  
     boost::barrier* noBarrier = nullptr;
     ThreadInitTask( m_threadInitTools, noBarrier, serviceLocator(), terminate )();
   }
  
   // Now, we do some error checking
   if ( ThreadInitTask::execFailed() ) {
     error() << "a ThreadInitTask failed to execute successfully" << endmsg;
     return StatusCode::FAILURE;
   }
  
   return StatusCode::SUCCESS;
 }

◆ poolSize()

int ThreadPoolSvc::poolSize ( ) const

inlinefinaloverride

Definition at line 55 of file ThreadPoolSvc.h.

55 { return m_threadPoolSize; }

◆ terminatePool()

StatusCode ThreadPoolSvc::terminatePool ( )

finaloverride

Terminate the thread pool and launch thread termination tasks.

Definition at line 137 of file ThreadPoolSvc.cpp.

                                         {
   tbb::spin_mutex::scoped_lock lock( m_initMutex );
  
   ON_DEBUG debug() << "ThreadPoolSvc::terminatePool()" << endmsg;
  
   if ( !m_init ) {
     error() << "Trying to terminate uninitialized thread pool!" << endmsg;
     return StatusCode::FAILURE;
   }
  
   // Launch the termination tasks
   const bool terminate = true;
   if ( launchTasks( terminate ).isFailure() ) return StatusCode::FAILURE;
  
   ON_DEBUG debug() << "Thread pool termination complete!" << endmsg;
  
   return StatusCode::SUCCESS;
 }

Member Data Documentation

◆ m_arena

tbb::task_arena ThreadPoolSvc::m_arena

private

TBB task arena to run all algorithms.

Definition at line 86 of file ThreadPoolSvc.h.

◆ m_barrier

std::unique_ptr<boost::barrier> ThreadPoolSvc::m_barrier

private

Barrier used to synchronization thread init tasks.

Definition at line 80 of file ThreadPoolSvc.h.

◆ m_init

bool ThreadPoolSvc::m_init = false

private

Was the thread pool initialized?

Definition at line 71 of file ThreadPoolSvc.h.

◆ m_initMutex

tbb::spin_mutex ThreadPoolSvc::m_initMutex

private

Mutex used to protect the initPool and terminatePool methods.

Definition at line 77 of file ThreadPoolSvc.h.

◆ m_tbbgc

std::unique_ptr<tbb::global_control> ThreadPoolSvc::m_tbbgc

private

TBB global control parameter.

Definition at line 83 of file ThreadPoolSvc.h.

◆ m_threadInitCount

std::atomic<int> ThreadPoolSvc::m_threadInitCount = 0

private

Counter for all threads that are initialised.

Definition at line 89 of file ThreadPoolSvc.h.

◆ m_threadInitTools

ToolHandleArray<IThreadInitTool> ThreadPoolSvc::m_threadInitTools = { this }

private

Handle array of thread init tools.

Definition at line 68 of file ThreadPoolSvc.h.

◆ m_threadPoolSize

int ThreadPoolSvc::m_threadPoolSize = 0

private

Size of the thread pool allocated.

Definition at line 74 of file ThreadPoolSvc.h.

The documentation for this class was generated from the following files:

GaudiHive/src/ThreadPoolSvc.h
GaudiHive/src/ThreadPoolSvc.cpp

Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ ThreadPoolSvc()

Member Function Documentation

◆ finalize()

◆ getArena()

◆ initialize()

◆ initPool()

◆ initThisThread()

◆ isInit()

◆ launchTasks()

◆ poolSize()

◆ terminatePool()

Member Data Documentation

◆ m_arena

◆ m_barrier

◆ m_init

◆ m_initMutex

◆ m_tbbgc

◆ m_threadInitCount

◆ m_threadInitTools

◆ m_threadPoolSize