allow workers to recurse on their own sched_mutex

get rid of *_if_prev_locked routines

src/common/thread.c

@@ -701,34 +701,47 @@ int starpu_pthread_barrier_wait(starpu_pthread_barrier_t *barrier)
  * macros of course) which record when the mutex is held or not */
 int starpu_pthread_mutex_lock_sched(starpu_pthread_mutex_t *mutex)
 {
-	int p_ret = starpu_pthread_mutex_lock(mutex);
-	int workerid = starpu_worker_get_id();
-	if(workerid != -1 && _starpu_worker_mutex_is_sched_mutex(workerid, mutex))
-		_starpu_worker_set_flag_sched_mutex_locked(workerid, 1);
-	return p_ret;
+	const int workerid = starpu_worker_get_id();
+	struct _starpu_worker * const worker = (workerid != -1)?_starpu_get_worker_struct(workerid):NULL;
+	if(worker && mutex == &worker->sched_mutex)
+	{
+		STARPU_ASSERT(worker->sched_mutex_depth < UINT_MAX);
+		worker->sched_mutex_depth++;
+		if (worker->sched_mutex_depth > 1)
+			return 0;
+	}
+
+	return starpu_pthread_mutex_lock(mutex);
 }
 
 int starpu_pthread_mutex_unlock_sched(starpu_pthread_mutex_t *mutex)
 {
-	int workerid = starpu_worker_get_id();
-	if(workerid != -1 && _starpu_worker_mutex_is_sched_mutex(workerid, mutex))
-		_starpu_worker_set_flag_sched_mutex_locked(workerid, 0);
+	const int workerid = starpu_worker_get_id();
+	struct _starpu_worker * const worker = (workerid != -1)?_starpu_get_worker_struct(workerid):NULL;
+	if(worker && mutex == &worker->sched_mutex)
+	{
+		STARPU_ASSERT(worker->sched_mutex_depth > 0);
+		worker->sched_mutex_depth--;
+		if (worker->sched_mutex_depth > 0)
+			return 0;
+	}
 
 	return starpu_pthread_mutex_unlock(mutex);
 }
 
 int starpu_pthread_mutex_trylock_sched(starpu_pthread_mutex_t *mutex)
 {
-	int ret = starpu_pthread_mutex_trylock(mutex);
-
-	if (!ret)
+	const int workerid = starpu_worker_get_id();
+	struct _starpu_worker * const worker = (workerid != -1)?_starpu_get_worker_struct(workerid):NULL;
+	if(worker && mutex == &worker->sched_mutex)
 	{
-		int workerid = starpu_worker_get_id();
-		if(workerid != -1 && _starpu_worker_mutex_is_sched_mutex(workerid, mutex))
-			_starpu_worker_set_flag_sched_mutex_locked(workerid, 1);
+		STARPU_ASSERT(worker->sched_mutex_depth < UINT_MAX);
+		worker->sched_mutex_depth++;
+		if (worker->sched_mutex_depth > 1)
+			return 0;
 	}
 
-	return ret;
+	return starpu_pthread_mutex_trylock(mutex);
 }
 
 #ifdef STARPU_DEBUG

src/core/sched_ctx.c

@@ -757,12 +757,10 @@ unsigned starpu_sched_ctx_create_inside_interval(const char *policy_name, const
 	sched_ctx->max_ncpus = max_ncpus;
 	sched_ctx->min_ngpus = min_ngpus;
 	sched_ctx->max_ngpus = max_ngpus;
-	_starpu_unlock_mutex_if_prev_locked();
 	int *added_workerids;
 	unsigned nw_ctx = starpu_sched_ctx_get_workers_list(sched_ctx->id, &added_workerids);
 	_starpu_update_workers_without_ctx(added_workerids, nw_ctx, sched_ctx->id, 0);
 	free(added_workerids);
-	_starpu_relock_mutex_if_prev_locked();
 #ifdef STARPU_USE_SC_HYPERVISOR
 	sched_ctx->perf_counters = NULL;
 #endif
@@ -907,12 +905,10 @@ unsigned starpu_sched_ctx_create(int *workerids, int nworkers, const char *sched
 	sched_ctx->hierarchy_level = hierarchy_level;
 	sched_ctx->nesting_sched_ctx = nesting_sched_ctx;
 
-	_starpu_unlock_mutex_if_prev_locked();
 	int *added_workerids;
 	unsigned nw_ctx = starpu_sched_ctx_get_workers_list(sched_ctx->id, &added_workerids);
 	_starpu_update_workers_with_ctx(added_workerids, nw_ctx, sched_ctx->id);
 	free(added_workerids);
-	_starpu_relock_mutex_if_prev_locked();
 #ifdef STARPU_USE_SC_HYPERVISOR
 	sched_ctx->perf_counters = NULL;
 #endif
@@ -1032,12 +1028,10 @@ int fstarpu_sched_ctx_create(int *workerids, int nworkers, const char *sched_ctx
 	sched_ctx->hierarchy_level = hierarchy_level;
 	sched_ctx->nesting_sched_ctx = nesting_sched_ctx;
 
-	_starpu_unlock_mutex_if_prev_locked();
 	int *added_workerids;
 	unsigned nw_ctx = starpu_sched_ctx_get_workers_list(sched_ctx->id, &added_workerids);
 	_starpu_update_workers_with_ctx(added_workerids, nw_ctx, sched_ctx->id);
 	free(added_workerids);
-	_starpu_relock_mutex_if_prev_locked();
 #ifdef STARPU_USE_SC_HYPERVISOR
 	sched_ctx->perf_counters = NULL;
 #endif
@@ -1124,7 +1118,6 @@ void starpu_sched_ctx_delete(unsigned sched_ctx_id)
 	unsigned inheritor_sched_ctx_id = sched_ctx->inheritor;
 	struct _starpu_sched_ctx *inheritor_sched_ctx = _starpu_get_sched_ctx_struct(sched_ctx->inheritor);
 
-	_starpu_unlock_mutex_if_prev_locked();
 	STARPU_PTHREAD_RWLOCK_WRLOCK(&changing_ctx_mutex[sched_ctx_id]);
 	STARPU_ASSERT(sched_ctx->id != STARPU_NMAX_SCHED_CTXS);
 
@@ -1178,7 +1171,6 @@ void starpu_sched_ctx_delete(unsigned sched_ctx_id)
 	/* workerids is malloc-ed in starpu_sched_ctx_get_workers_list, don't forget to free it when
 	   you don't use it anymore */
 	free(workerids);
-	_starpu_relock_mutex_if_prev_locked();
 	occupied_sms -= sched_ctx->nsms;
 	return;
 }
@@ -1350,8 +1342,6 @@ void starpu_sched_ctx_add_workers(int *workers_to_add, int nworkers_to_add, unsi
 {
 	struct _starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx_struct(sched_ctx_id);
 
-	_starpu_unlock_mutex_if_prev_locked();
-
 	STARPU_PTHREAD_RWLOCK_WRLOCK(&changing_ctx_mutex[sched_ctx_id]);
 
 	STARPU_ASSERT(workers_to_add != NULL && nworkers_to_add > 0);
@@ -1376,8 +1366,6 @@ void starpu_sched_ctx_add_workers(int *workers_to_add, int nworkers_to_add, unsi
 
 	STARPU_PTHREAD_RWLOCK_UNLOCK(&changing_ctx_mutex[sched_ctx_id]);
 
-	_starpu_relock_mutex_if_prev_locked();
-
 	if(sched_ctx->id != STARPU_NMAX_SCHED_CTXS)
 	{
 		STARPU_PTHREAD_RWLOCK_RDLOCK(&changing_ctx_mutex[sched_ctx_id]);
@@ -1394,8 +1382,6 @@ void starpu_sched_ctx_remove_workers(int *workers_to_remove, int nworkers_to_rem
 
 	_starpu_check_workers(workers_to_remove, nworkers_to_remove);
 
-	_starpu_unlock_mutex_if_prev_locked();
-
 	STARPU_PTHREAD_RWLOCK_WRLOCK(&changing_ctx_mutex[sched_ctx_id]);
 	/* if the context has not already been deleted */
 	if(sched_ctx->id != STARPU_NMAX_SCHED_CTXS)
@@ -1414,8 +1400,6 @@ void starpu_sched_ctx_remove_workers(int *workers_to_remove, int nworkers_to_rem
 	}
 	STARPU_PTHREAD_RWLOCK_UNLOCK(&changing_ctx_mutex[sched_ctx_id]);
 
-	_starpu_relock_mutex_if_prev_locked();
-
 	return;
 }
 

src/core/workers.c

@@ -576,7 +576,7 @@ static void _starpu_worker_init(struct _starpu_worker *workerarg, struct _starpu
 	workerarg->reverse_phase[0] = 0;
 	workerarg->reverse_phase[1] = 0;
 	workerarg->pop_ctx_priority = 1;
-	workerarg->sched_mutex_locked = 0;
+	workerarg->sched_mutex_depth = 0;
 	workerarg->blocked = 0;
 	workerarg->is_slave_somewhere = 0;
 
@@ -2161,33 +2161,6 @@ void starpu_get_version(int *major, int *minor, int *release)
 	*release = STARPU_RELEASE_VERSION;
 }
 
-void _starpu_unlock_mutex_if_prev_locked()
-{
-	int workerid = starpu_worker_get_id();
-	if(workerid != -1)
-	{
-		struct _starpu_worker *w = _starpu_get_worker_struct(workerid);
-		if(w->sched_mutex_locked)
-		{
-			STARPU_PTHREAD_MUTEX_UNLOCK_SCHED(&w->sched_mutex);
-			_starpu_worker_set_flag_sched_mutex_locked(workerid, 1);
-		}
-	}
-	return;
-}
-
-void _starpu_relock_mutex_if_prev_locked()
-{
-	int workerid = starpu_worker_get_id();
-	if(workerid != -1)
-	{
-		struct _starpu_worker *w = _starpu_get_worker_struct(workerid);
-		if(w->sched_mutex_locked)
-			STARPU_PTHREAD_MUTEX_LOCK_SCHED(&w->sched_mutex);
-	}
-	return;
-}
-
 unsigned starpu_worker_get_sched_ctx_list(int workerid, unsigned **sched_ctxs)
 {
 	unsigned s = 0;

src/core/workers.h

@@ -137,8 +137,8 @@ LIST_TYPE(_starpu_worker,
 	/* indicate which priority of ctx is currently active: the values are 0 or 1*/
 	unsigned pop_ctx_priority;
 
-	/* flag to know if sched_mutex is locked or not */
-	unsigned sched_mutex_locked;
+	/* sched mutex local worker locking depth */
+	unsigned sched_mutex_depth;
 
 	/* bool to indicate if the worker is blocked in a ctx */
 	unsigned blocked;
@@ -558,18 +558,6 @@ int starpu_worker_get_nids_by_type(enum starpu_worker_archtype type, int *worker
    the list might not be updated */
 int starpu_worker_get_nids_ctx_free_by_type(enum starpu_worker_archtype type, int *workerids, int maxsize);
 
-/* if the current worker has the lock release it */
-void _starpu_unlock_mutex_if_prev_locked();
-
-/* if we prev released the lock relock it */
-void _starpu_relock_mutex_if_prev_locked();
-
-static inline void _starpu_worker_set_flag_sched_mutex_locked(int workerid, unsigned flag)
-{
-	struct _starpu_worker *w = _starpu_get_worker_struct(workerid);
-	w->sched_mutex_locked = flag;
-}
-
 static inline unsigned _starpu_worker_mutex_is_sched_mutex(int workerid, starpu_pthread_mutex_t *mutex)
 {
 	struct _starpu_worker *w = _starpu_get_worker_struct(workerid);