Use starpu_worker_relax_on instead of internal _starpu_worker_relax_on, for coherency

src/common/graph.c

@@ -63,9 +63,9 @@ void _starpu_graph_init(void)
 /* LockWR the graph lock */
 void _starpu_graph_wrlock(void)
 {
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_RWLOCK_WRLOCK(&graph_lock);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 }
 
 void _starpu_graph_drop_node(struct _starpu_graph_node *node);
@@ -99,18 +99,18 @@ void _starpu_graph_drop_dropped_nodes(void)
 /* UnlockWR the graph lock */
 void _starpu_graph_wrunlock(void)
 {
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&dropped_lock);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 	_starpu_graph_drop_dropped_nodes();
 }
 
 /* LockRD the graph lock */
 void _starpu_graph_rdlock(void)
 {
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_RWLOCK_RDLOCK(&graph_lock);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 }
 
 /* UnlockRD the graph lock */
@@ -256,16 +256,16 @@ void _starpu_graph_drop_job(struct _starpu_job *job)
 	if (!node)
 		return;
 
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&node->mutex);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 	/* Will not be able to use the job any more */
 	node->job = NULL;
 	STARPU_PTHREAD_MUTEX_UNLOCK(&node->mutex);
 
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&dropped_lock);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 	/* Queue for removal when lock becomes available */
 	_starpu_graph_node_multilist_push_back_dropped(&dropped, node);
 	if (STARPU_PTHREAD_RWLOCK_TRYWRLOCK(&graph_lock) == 0)

src/core/workers.c

@@ -2543,7 +2543,7 @@ int starpu_wake_worker_relax_light(int workerid)
 	int cur_workerid = starpu_worker_get_id();
 	if (workerid != cur_workerid)
 	{
-		_starpu_worker_relax_on();
+		starpu_worker_relax_on();
 
 		STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->sched_mutex);
 		while (!worker->state_relax_refcnt)
@@ -2553,7 +2553,7 @@ int starpu_wake_worker_relax_light(int workerid)
 			{
 				_starpu_config.workers[workerid].state_keep_awake = 1;
 				STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);
-				_starpu_worker_relax_off();
+				starpu_worker_relax_off();
 				return 1;
 			}
 
@@ -2568,7 +2568,7 @@ int starpu_wake_worker_relax_light(int workerid)
 	STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&worker->sched_mutex);
 	if (workerid != cur_workerid)
 	{
-		_starpu_worker_relax_off();
+		starpu_worker_relax_off();
 	}
 	return ret;
 }

src/core/workers.h

@@ -1103,7 +1103,7 @@ static inline void _starpu_worker_lock(int workerid)
 	int cur_workerid = starpu_worker_get_id();
 	if (workerid != cur_workerid)
 	{
-		_starpu_worker_relax_on();
+		starpu_worker_relax_on();
 
 		STARPU_PTHREAD_MUTEX_LOCK_SCHED(&worker->sched_mutex);
 		while (!worker->state_relax_refcnt)
@@ -1155,7 +1155,7 @@ static inline void _starpu_worker_unlock(int workerid)
 	int cur_workerid = starpu_worker_get_id();
 	if (workerid != cur_workerid)
 	{
-		_starpu_worker_relax_off();
+		starpu_worker_relax_off();
 	}
 }
 

src/sched_policies/deque_modeling_policy_data_aware.c

@@ -351,9 +351,9 @@ static int push_task_on_best_worker(struct starpu_task *task, int best_workerid,
 
 	if (_starpu_get_nsched_ctxs() > 1)
 	{
-		_starpu_worker_relax_on();
+		starpu_worker_relax_on();
 		_starpu_sched_ctx_lock_write(sched_ctx_id);
-		_starpu_worker_relax_off();
+		starpu_worker_relax_off();
 		if (_starpu_sched_ctx_worker_is_master_for_child_ctx(sched_ctx_id, best_workerid, task))
 			task = NULL;
 		_starpu_sched_ctx_unlock_write(sched_ctx_id);
@@ -437,9 +437,9 @@ static int push_task_on_best_worker(struct starpu_task *task, int best_workerid,
 		unsigned stream_ctx_id = starpu_worker_get_sched_ctx_id_stream(best_workerid);
 		if(stream_ctx_id != STARPU_NMAX_SCHED_CTXS)
 		{
-			_starpu_worker_relax_on();
+			starpu_worker_relax_on();
 			_starpu_sched_ctx_lock_write(sched_ctx_id);
-			_starpu_worker_relax_off();
+			starpu_worker_relax_off();
 			starpu_sched_ctx_move_task_to_ctx_locked(task, stream_ctx_id, 0);
 			starpu_sched_ctx_revert_task_counters_ctx_locked(sched_ctx_id, task->flops);
 			_starpu_sched_ctx_unlock_write(sched_ctx_id);

src/sched_policies/eager_central_policy.c

@@ -74,18 +74,18 @@ static int push_task_eager_policy(struct starpu_task *task)
 	unsigned sched_ctx_id = task->sched_ctx;
 	struct _starpu_eager_center_policy_data *data = (struct _starpu_eager_center_policy_data*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
 
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&data->policy_mutex);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 	starpu_task_list_push_back(&data->fifo->taskq,task);
 	data->fifo->ntasks++;
 	data->fifo->nprocessed++;
 
 	if (_starpu_get_nsched_ctxs() > 1)
 	{
-		_starpu_worker_relax_on();
+		starpu_worker_relax_on();
 		_starpu_sched_ctx_lock_write(sched_ctx_id);
-		_starpu_worker_relax_off();
+		starpu_worker_relax_off();
 		starpu_sched_ctx_list_task_counters_increment_all_ctx_locked(task, sched_ctx_id);
 		_starpu_sched_ctx_unlock_write(sched_ctx_id);
 	}
@@ -178,9 +178,9 @@ static struct starpu_task *pop_task_eager_policy(unsigned sched_ctx_id)
 	}
 #endif
 	/* block until some event happens */
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&data->policy_mutex);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 
 	chosen_task = _starpu_fifo_pop_task(data->fifo, workerid);
 	if (!chosen_task)
@@ -190,9 +190,9 @@ static struct starpu_task *pop_task_eager_policy(unsigned sched_ctx_id)
 	STARPU_PTHREAD_MUTEX_UNLOCK(&data->policy_mutex);
 	if(chosen_task &&_starpu_get_nsched_ctxs() > 1)
 	{
-		_starpu_worker_relax_on();
+		starpu_worker_relax_on();
 		_starpu_sched_ctx_lock_write(sched_ctx_id);
-		_starpu_worker_relax_off();
+		starpu_worker_relax_off();
 		starpu_sched_ctx_list_task_counters_decrement_all_ctx_locked(chosen_task, sched_ctx_id);
 
 		if (_starpu_sched_ctx_worker_is_master_for_child_ctx(sched_ctx_id, workerid, chosen_task))

src/sched_policies/eager_central_priority_policy.c

@@ -86,16 +86,16 @@ static int _starpu_priority_push_task(struct starpu_task *task)
 	struct _starpu_eager_central_prio_data *data = (struct _starpu_eager_central_prio_data*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
 	struct _starpu_prio_deque *taskq = &data->taskq;
 
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&data->policy_mutex);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 	_starpu_prio_deque_push_back_task(taskq, task);
 
 	if (_starpu_get_nsched_ctxs() > 1)
 	{
-		_starpu_worker_relax_on();
+		starpu_worker_relax_on();
 		_starpu_sched_ctx_lock_write(sched_ctx_id);
-		_starpu_worker_relax_off();
+		starpu_worker_relax_off();
 		starpu_sched_ctx_list_task_counters_increment_all_ctx_locked(task, sched_ctx_id);
 		_starpu_sched_ctx_unlock_write(sched_ctx_id);
 	}
@@ -179,9 +179,9 @@ static struct starpu_task *_starpu_priority_pop_task(unsigned sched_ctx_id)
 	}
 #endif
 	/* block until some event happens */
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&data->policy_mutex);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 
 	chosen_task = _starpu_prio_deque_pop_task_for_worker(taskq, workerid, &skipped);
 
@@ -215,9 +215,9 @@ static struct starpu_task *_starpu_priority_pop_task(unsigned sched_ctx_id)
 	STARPU_PTHREAD_MUTEX_UNLOCK(&data->policy_mutex);
 	if(chosen_task &&_starpu_get_nsched_ctxs() > 1)
 	{
-		_starpu_worker_relax_on();
+		starpu_worker_relax_on();
 		_starpu_sched_ctx_lock_write(sched_ctx_id);
-		_starpu_worker_relax_off();
+		starpu_worker_relax_off();
 		starpu_sched_ctx_list_task_counters_decrement_all_ctx_locked(chosen_task, sched_ctx_id);
 
 		if (_starpu_sched_ctx_worker_is_master_for_child_ctx(sched_ctx_id, workerid, chosen_task))

src/sched_policies/graph_test_policy.c

@@ -160,9 +160,9 @@ static struct _starpu_prio_deque *select_prio(unsigned sched_ctx_id, struct _sta
 static void set_priority(void *_data, struct _starpu_graph_node *node)
 {
 	struct _starpu_graph_test_policy_data *data = _data;
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&node->mutex);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 	struct _starpu_job *job = node->job;
 	if (job)
 	{
@@ -178,9 +178,9 @@ static void do_schedule_graph_test_policy(unsigned sched_ctx_id)
 {
 	struct _starpu_graph_test_policy_data *data = (struct _starpu_graph_test_policy_data*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
 
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&data->policy_mutex);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 	if (data->descendants)
 		_starpu_graph_compute_descendants();
 	else
@@ -233,9 +233,9 @@ static int push_task_graph_test_policy(struct starpu_task *task)
 	unsigned sched_ctx_id = task->sched_ctx;
 	struct _starpu_graph_test_policy_data *data = (struct _starpu_graph_test_policy_data*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
 
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&data->policy_mutex);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 	if (!data->computed)
 	{
 		/* Priorities are not computed, leave the task in the bag for now */
@@ -340,9 +340,9 @@ static struct starpu_task *pop_task_graph_test_policy(unsigned sched_ctx_id)
 		return NULL;
 #endif
 
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&data->policy_mutex);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 	if (!data->computed)
 	{
 		STARPU_PTHREAD_MUTEX_UNLOCK(&data->policy_mutex);

src/sched_policies/heteroprio.c

@@ -376,9 +376,9 @@ static int push_task_heteroprio_policy(struct starpu_task *task)
 	struct _starpu_heteroprio_data *hp = (struct _starpu_heteroprio_data*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
 
 	/* One worker at a time use heteroprio */
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&hp->policy_mutex);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 
 	/* Retrieve the correct bucket */
 	STARPU_ASSERT(task->priority < STARPU_HETEROPRIO_MAX_PRIO);
@@ -475,9 +475,9 @@ static struct starpu_task *pop_task_heteroprio_policy(unsigned sched_ctx_id)
 		return NULL;
 	}
 #endif
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&hp->policy_mutex);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 
 	/* keep track of the new added task to perfom real prefetch on node */
 	unsigned nb_added_tasks = 0;
@@ -622,9 +622,9 @@ done:		;
 
 	if(task &&_starpu_get_nsched_ctxs() > 1)
 	{
-		_starpu_worker_relax_on();
+		starpu_worker_relax_on();
 		_starpu_sched_ctx_lock_write(sched_ctx_id);
-		_starpu_worker_relax_off();
+		starpu_worker_relax_off();
 		if (_starpu_sched_ctx_worker_is_master_for_child_ctx(sched_ctx_id, workerid, task))
 			task = NULL;
 		_starpu_sched_ctx_unlock_write(sched_ctx_id);

src/sched_policies/parallel_eager.c

@@ -250,9 +250,9 @@ static struct starpu_task *pop_task_peager_policy(unsigned sched_ctx_id)
 	if (starpu_worker_get_type(workerid) != STARPU_CPU_WORKER && starpu_worker_get_type(workerid) != STARPU_MIC_WORKER)
 	{
 		struct starpu_task *task = NULL;
-		_starpu_worker_relax_on();
+		starpu_worker_relax_on();
 		STARPU_PTHREAD_MUTEX_LOCK(&data->policy_mutex);
-		_starpu_worker_relax_off();
+		starpu_worker_relax_off();
 		task = _starpu_fifo_pop_task(data->fifo, workerid);
 		STARPU_PTHREAD_MUTEX_UNLOCK(&data->policy_mutex);
 
@@ -261,9 +261,9 @@ static struct starpu_task *pop_task_peager_policy(unsigned sched_ctx_id)
 
 	struct starpu_task *task = NULL;
 	int slave_task = 0;
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	STARPU_PTHREAD_MUTEX_LOCK(&data->policy_mutex);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 	/* check if a slave task is available in the local queue */
 	task = _starpu_fifo_pop_task(data->local_fifo[workerid], workerid);
 	if (!task)

src/sched_policies/parallel_heft.c

@@ -127,9 +127,9 @@ static int push_task_on_best_worker(struct starpu_task *task, int best_workerid,
 
 		/* We don't want it to interlace its task with a combined
 		 * worker's one */
-		_starpu_worker_relax_on();
+		starpu_worker_relax_on();
 		STARPU_PTHREAD_MUTEX_LOCK(&hd->global_push_mutex);
-		_starpu_worker_relax_off();
+		starpu_worker_relax_off();
 
 		ret = starpu_push_local_task(best_workerid, task, prio);
 
@@ -149,9 +149,9 @@ static int push_task_on_best_worker(struct starpu_task *task, int best_workerid,
 		starpu_combined_worker_get_description(best_workerid, &worker_size, &combined_workerid);
 
 		/* All cpu workers must be locked at once */
-		_starpu_worker_relax_on();
+		starpu_worker_relax_on();
 		STARPU_PTHREAD_MUTEX_LOCK(&hd->global_push_mutex);
-		_starpu_worker_relax_off();
+		starpu_worker_relax_off();
 
 		/* This is a combined worker so we create task aliases */
 		int i;

src/sched_policies/work_stealing_policy.c

@@ -549,9 +549,9 @@ static struct starpu_task *ws_pop_task(unsigned sched_ctx_id)
 		ws->per_worker[workerid].busy = 1;
 		if (_starpu_get_nsched_ctxs() > 1)
 		{
-			_starpu_worker_relax_on();
+			starpu_worker_relax_on();
 			_starpu_sched_ctx_lock_write(sched_ctx_id);
-			_starpu_worker_relax_off();
+			starpu_worker_relax_off();
 			starpu_sched_ctx_list_task_counters_decrement(sched_ctx_id, workerid);
 			if (_starpu_sched_ctx_worker_is_master_for_child_ctx(sched_ctx_id, workerid, task))
 				task = NULL;
@@ -561,9 +561,9 @@ static struct starpu_task *ws_pop_task(unsigned sched_ctx_id)
 	}
 
 	/* we need to steal someone's job */
-	_starpu_worker_relax_on();
+	starpu_worker_relax_on();
 	int victim = ws->select_victim(ws, sched_ctx_id, workerid);
-	_starpu_worker_relax_off();
+	starpu_worker_relax_off();
 	if (victim == -1)
 	{
 		return NULL;
@@ -610,9 +610,9 @@ static struct starpu_task *ws_pop_task(unsigned sched_ctx_id)
 
 	if (task &&_starpu_get_nsched_ctxs() > 1)
 	{
-		_starpu_worker_relax_on();
+		starpu_worker_relax_on();
 		_starpu_sched_ctx_lock_write(sched_ctx_id);
-		_starpu_worker_relax_off();
+		starpu_worker_relax_off();
 		if (_starpu_sched_ctx_worker_is_master_for_child_ctx(sched_ctx_id, workerid, task))
 			task = NULL;
 		_starpu_sched_ctx_unlock_write(sched_ctx_id);