add optional callbacks to notify an external resource manager about workers going to sleep and waking up

configure.ac

@@ -235,6 +235,20 @@ if test x$enable_blocking = xno ; then
 	AC_DEFINE(STARPU_NON_BLOCKING_DRIVERS, [1], [drivers must progress])
 fi
 
+if test x$enable_blocking = xyes ; then
+AC_MSG_CHECKING(whether worker callbacks should be enabled)
+AC_ARG_ENABLE(worker-callbacks, [AS_HELP_STRING([--enable-worker-callbacks], [enable worker callbacks])],
+				enable_worker_cb=$enableval, enable_worker_cb=no)
+AC_MSG_RESULT($enable_worker_cb)
+else
+	# worker sleep/wake-up callbacks only make sense if blocking drivers are enabled
+	enable_worker_cb=no
+fi
+
+if test x$enable_worker_cb = xyes ; then
+	AC_DEFINE(STARPU_WORKER_CALLBACKS, [1], [workers must call callbacks on sleep/wake-up])
+fi
+
 ###############################################################################
 #                                                                             #
 #                                    MPI                                      #

doc/doxygen/chapters/510_configure_options.doxy

@@ -140,6 +140,14 @@ reactivity. This option makes StarPU put CPU workers to real sleep when there
 are not enough tasks to compute.
 </dd>
 
+<dt>--enable-worker-callbacks</dt>
+<dd>
+\anchor enable-worker-callbacks
+\addindex __configure__--enable-worker-callbacks
+If blocking drivers are enabled, enable callbacks to notify an external resource manager
+about workers going to sleep and waking up.
+</dd>
+
 <dt>--enable-maxcpus=<c>count</c></dt>
 <dd>
 \anchor enable-maxcpus

doc/doxygen/chapters/api/initialization.doxy

@@ -228,6 +228,16 @@ The environment variables overwrite the equivalent parameters.
 \var starpu_conf::global_sched_ctx_max_priority
     todo
 
+\var starpu_conf::callback_worker_going_to_sleep
+    If StarPU was compiler with blocking drivers support and worker
+    callbacks support enabled, allow to specify an external resource
+    manager callback to be notified about workers going to sleep.
+
+\var starpu_conf::callback_worker_waking_up
+    If StarPU was compiler with blocking drivers support and worker
+    callbacks support enabled, allow to specify an external resource
+    manager callback to be notified about workers waking-up.
+
 \fn int starpu_init(struct starpu_conf *conf)
 \ingroup API_Initialization_and_Termination
 This is StarPU initialization method, which must be called prior to

doc/doxygen/chapters/api/workers.doxy

@@ -342,4 +342,16 @@ hwloc cpuset associated with the worker \p workerid. The returned cpuset is obta
 from a \c hwloc_bitmap_dup() function call. It must be freed by the caller
 using \c hwloc_bitmap_free().
 
+\fn void starpu_worker_set_going_to_sleep_callback(void (*callback)(unsigned workerid))
+\ingroup API_Workers_Properties
+If StarPU was compiler with blocking drivers support and worker callbacks support
+enabled, allow to specify an external resource manager callback to be notified
+about workers going to sleep.
+
+\fn void starpu_worker_set_waking_up_callback(void (*callback)(unsigned workerid))
+\ingroup API_Workers_Properties
+If StarPU was compiler with blocking drivers support and worker callbacks support
+enabled, allow to specify an external resource manager callback to be notified
+about workers waking-up.
+
 */

include/starpu.h

@@ -140,6 +140,11 @@ struct starpu_conf
 	unsigned trace_buffer_size;
 	int global_sched_ctx_min_priority;
 	int global_sched_ctx_max_priority;
+
+#ifdef STARPU_WORKER_CALLBACKS
+	void (*callback_worker_going_to_sleep)(unsigned workerid);
+	void (*callback_worker_waking_up)(unsigned workerid);
+#endif
 };
 
 int starpu_conf_init(struct starpu_conf *conf);

include/starpu_config.h.in

@@ -49,6 +49,9 @@
 #undef STARPU_VALGRIND_FULL
 #undef STARPU_SANITIZE_LEAK
 #undef STARPU_NON_BLOCKING_DRIVERS
+/* workers must call callbacks on sleep/wake-up */
+#undef STARPU_WORKER_CALLBACKS
+
 
 #undef STARPU_HAVE_ICC
 

include/starpu_worker.h

@@ -166,6 +166,12 @@ void starpu_worker_unlock_self(void);
 
 int starpu_wake_worker_relax(int workerid);
 
+#ifdef STARPU_WORKER_CALLBACKS
+void starpu_worker_set_going_to_sleep_callback(void (*callback)(unsigned workerid));
+
+void starpu_worker_set_waking_up_callback(void (*callback)(unsigned workerid));
+#endif
+
 #ifdef STARPU_HAVE_HWLOC
 hwloc_cpuset_t starpu_worker_get_hwloc_cpuset(int workerid);
 #endif

src/core/workers.c

@@ -2529,3 +2529,17 @@ int _starpu_wake_worker_relax_light(int workerid)
 	}
 	return ret;
 }
+
+#ifdef STARPU_WORKER_CALLBACKS
+void starpu_worker_set_going_to_sleep_callback(void (*callback)(unsigned workerid))
+{
+	STARPU_ASSERT(_starpu_config.conf.callback_worker_going_to_sleep);
+	_starpu_config.conf.callback_worker_going_to_sleep = callback;
+}
+
+void starpu_worker_set_waking_up_callback(void (*callback)(unsigned workerid))
+{
+	STARPU_ASSERT(_starpu_config.conf.callback_worker_waking_up);
+	_starpu_config.conf.callback_worker_waking_up = callback;
+}
+#endif

src/drivers/driver_common/driver_common.c

@@ -396,6 +396,13 @@ struct starpu_task *_starpu_get_worker_task(struct _starpu_worker *worker, int w
 			&& !worker->state_unblock_in_parallel_req
 			&& !_starpu_sched_ctx_last_worker_awake(worker))
 		{
+
+#ifdef STARPU_WORKER_CALLBACKS
+			if (_starpu_config.conf.callback_worker_going_to_sleep != NULL)
+			{
+				_starpu_config.conf.callback_worker_going_to_sleep(workerid);
+			}
+#endif
 			do
 			{
 				STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
@@ -416,6 +423,12 @@ struct starpu_task *_starpu_get_worker_task(struct _starpu_worker *worker, int w
 				}
 			}
 			while (1);
+#ifdef STARPU_WORKER_CALLBACKS
+			if (_starpu_config.conf.callback_worker_waking_up != NULL)
+			{
+				_starpu_config.conf.callback_worker_waking_up(workerid);
+			}
+#endif
 			worker->state_keep_awake = 0;
 			_starpu_worker_set_status_scheduling_done(workerid);
 			STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);
@@ -591,6 +604,12 @@ int _starpu_get_multi_worker_task(struct _starpu_worker *workers, struct starpu_
 			&& !worker->state_unblock_in_parallel_req
 			&& !_starpu_sched_ctx_last_worker_awake(worker))
 		{
+#ifdef STARPU_WORKER_CALLBACKS
+			if (_starpu_config.conf.callback_worker_going_to_sleep != NULL)
+			{
+				_starpu_config.conf.callback_worker_going_to_sleep(workerid);
+			}
+#endif
 			do
 			{
 				STARPU_PTHREAD_COND_WAIT(&worker->sched_cond, &worker->sched_mutex);
@@ -611,6 +630,12 @@ int _starpu_get_multi_worker_task(struct _starpu_worker *workers, struct starpu_
 				}
 			}
 			while (1);
+#ifdef STARPU_WORKER_CALLBACKS
+			if (_starpu_config.conf.callback_worker_waking_up != NULL)
+			{
+				_starpu_config.conf.callback_worker_waking_up(workerid);
+			}
+#endif
 			worker->state_keep_awake = 0;
 			_starpu_worker_set_status_scheduling_done(workerid);
 			STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);