do not take changing_ctx_mutex when finishing executing the tasks (in decrement_nsubmitted_tasks), just check if the counter is 1, do what you have to do, and then decrement it (decrement the counter while verifying it wakes up the the thread calling wait_for_all)

src/core/sched_ctx.c

@@ -818,7 +818,8 @@ void _starpu_decrement_nsubmitted_tasks_of_sched_ctx(unsigned sched_ctx_id)
 		config->watchdog_ok = 1;
 
 	struct _starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx_struct(sched_ctx_id);
-	int finished = _starpu_barrier_counter_decrement_until_empty_counter(&sched_ctx->tasks_barrier, 0.0);
+	int reached = _starpu_barrier_counter_get_reached_start(&sched_ctx->tasks_barrier);
+	int finished = reached == 1;
         /* when finished decrementing the tasks if the user signaled he will not submit tasks anymore
            we can move all its workers to the inheritor context */
 	if(finished && sched_ctx->inheritor != STARPU_NMAX_SCHED_CTXS)
@@ -828,8 +829,6 @@ void _starpu_decrement_nsubmitted_tasks_of_sched_ctx(unsigned sched_ctx_id)
 		{
 			STARPU_PTHREAD_MUTEX_UNLOCK(&finished_submit_mutex);
 
-			/* take care the context is not deleted or changed at the same time */
-			STARPU_PTHREAD_RWLOCK_RDLOCK(&changing_ctx_mutex[sched_ctx_id]);
 			if(sched_ctx->id != STARPU_NMAX_SCHED_CTXS)
 			{
 				if(sched_ctx->close_callback)
@@ -844,8 +843,7 @@ void _starpu_decrement_nsubmitted_tasks_of_sched_ctx(unsigned sched_ctx_id)
 					free(workerids);
 				}
 			}
-			STARPU_PTHREAD_RWLOCK_UNLOCK(&changing_ctx_mutex[sched_ctx_id]);
-
+			_starpu_barrier_counter_decrement_until_empty_counter(&sched_ctx->tasks_barrier, 0.0);
 			return;
 		}
 		STARPU_PTHREAD_MUTEX_UNLOCK(&finished_submit_mutex);
@@ -861,13 +859,11 @@ void _starpu_decrement_nsubmitted_tasks_of_sched_ctx(unsigned sched_ctx_id)
 	STARPU_PTHREAD_MUTEX_LOCK(&config->submitted_mutex);
 	if(config->submitting == 0)
 	{
-		STARPU_PTHREAD_RWLOCK_RDLOCK(&changing_ctx_mutex[sched_ctx_id]);
 		if(sched_ctx->id != STARPU_NMAX_SCHED_CTXS)
 		{
 			if(sched_ctx->close_callback)
 				sched_ctx->close_callback(sched_ctx->id, sched_ctx->close_args);
 		}
-		STARPU_PTHREAD_RWLOCK_UNLOCK(&changing_ctx_mutex[sched_ctx_id]);
 
 		ANNOTATE_HAPPENS_AFTER(&config->running);
 		config->running = 0;
@@ -883,6 +879,8 @@ void _starpu_decrement_nsubmitted_tasks_of_sched_ctx(unsigned sched_ctx_id)
 	}
 	STARPU_PTHREAD_MUTEX_UNLOCK(&config->submitted_mutex);
 
+	_starpu_barrier_counter_decrement_until_empty_counter(&sched_ctx->tasks_barrier, 0.0);
+
 	return;
 }