fix hypervisor to better react to resizing requirements + fix last release small issues (compiling issues in the doc & a small issue in datawizard)

doc/chapters/perf-optimization.texi

@@ -582,15 +582,15 @@ CUDA or OpenCL execution:
 
 @smallexample
 static struct starpu_codelet cl11 =
-{
-	.cpu_funcs = {chol_cpu_codelet_update_u11, NULL},
+@{
+	.cpu_funcs = @{chol_cpu_codelet_update_u11, NULL@},
 #ifdef STARPU_USE_CUDA
-	.cuda_funcs = {chol_cublas_codelet_update_u11, NULL},
+	.cuda_funcs = @{chol_cublas_codelet_update_u11, NULL@},
 #elif defined(STARPU_SIMGRID)
-	.cuda_funcs = {(void*)1, NULL},
+	.cuda_funcs = @{(void*)1, NULL@},
 #endif
 	.nbuffers = 1,
-	.modes = {STARPU_RW},
+	.modes = @{STARPU_RW@},
 	.model = &chol_model_11
-};
+@};
 @end smallexample

sched_ctx_hypervisor/include/sched_ctx_hypervisor.h

@@ -151,7 +151,7 @@ int *sched_ctx_hypervisor_get_sched_ctxs();
 
 int sched_ctx_hypervisor_get_nsched_ctxs();
 
-int get_nworkers_ctx(unsigned sched_ctx, enum starpu_archtype arch);
+int sched_ctx_hypervisor_get_nworkers_ctx(unsigned sched_ctx, enum starpu_archtype arch);
 
 struct sched_ctx_hypervisor_wrapper *sched_ctx_hypervisor_get_wrapper(unsigned sched_ctx);
 

sched_ctx_hypervisor/src/hypervisor_policies/lp_tools.c

@@ -289,7 +289,7 @@ void _lp_redistribute_resources_in_ctxs(int ns, int nw, int res_rounded[ns][nw],
 
 			if(w == 1)
 			{
-				int nworkers_ctx = get_nworkers_ctx(sched_ctxs[s], arch);
+				int nworkers_ctx = sched_ctx_hypervisor_get_nworkers_ctx(sched_ctxs[s], arch);
 				if(nworkers_ctx > res_rounded[s][w])
 				{
 					int nworkers_to_move = nworkers_ctx - res_rounded[s][w];
@@ -302,7 +302,7 @@ void _lp_redistribute_resources_in_ctxs(int ns, int nw, int res_rounded[ns][nw],
 			}
 			else
 			{
-				double nworkers_ctx = get_nworkers_ctx(sched_ctxs[s], arch) * 1.0;
+				double nworkers_ctx = sched_ctx_hypervisor_get_nworkers_ctx(sched_ctxs[s], arch) * 1.0;
 				if(nworkers_ctx > res[s][w])
 				{
 					double nworkers_to_move = nworkers_ctx - res[s][w];
@@ -347,7 +347,7 @@ void _lp_redistribute_resources_in_ctxs(int ns, int nw, int res_rounded[ns][nw],
 			{
 				if(sched_ctxs[s2] != sched_ctxs[s])
 				{
-					double nworkers_ctx2 = get_nworkers_ctx(sched_ctxs[s2], arch) * 1.0;
+					double nworkers_ctx2 = sched_ctx_hypervisor_get_nworkers_ctx(sched_ctxs[s2], arch) * 1.0;
 					if((res[s2][w] - nworkers_ctx2) >= 0.0 && nw_move > 0)
 					{
 						sched_ctx_hypervisor_move_workers(sched_ctxs[s], sched_ctxs[s2], workers_move, nw_move, 0);

sched_ctx_hypervisor/src/hypervisor_policies/policy_tools.c

@@ -327,8 +327,7 @@ double _get_ctx_velocity(struct sched_ctx_hypervisor_wrapper* sc_w)
         double elapsed_flops = sched_ctx_hypervisor_get_elapsed_flops_per_sched_ctx(sc_w);
 	double total_elapsed_flops = sched_ctx_hypervisor_get_total_elapsed_flops_per_sched_ctx(sc_w);
 	double prc = elapsed_flops/sc_w->total_flops;
-	unsigned nworkers = starpu_sched_ctx_get_nworkers(sc_w->sched_ctx);
-	double redim_sample = elapsed_flops == total_elapsed_flops ? HYPERVISOR_START_REDIM_SAMPLE*nworkers : HYPERVISOR_REDIM_SAMPLE*nworkers;
+	double redim_sample = elapsed_flops == total_elapsed_flops ? HYPERVISOR_START_REDIM_SAMPLE : HYPERVISOR_REDIM_SAMPLE;
 	if(prc >= redim_sample)
         {
                 double curr_time = starpu_timing_now();
@@ -374,6 +373,10 @@ int _velocity_gap_btw_ctxs()
 			{
 				if(sched_ctxs[i] != sched_ctxs[j])
 				{
+					unsigned nworkers = starpu_sched_ctx_get_nworkers(sched_ctxs[j]);
+					if(nworkers == 0) 
+						return 1;
+
 					other_sc_w = sched_ctx_hypervisor_get_wrapper(sched_ctxs[j]);
 					double other_ctx_v = _get_ctx_velocity(other_sc_w);
 					if(other_ctx_v != 0.0)
@@ -382,8 +385,6 @@ int _velocity_gap_btw_ctxs()
 						if(gap > 1.5)
 							return 1;
 					}
-					else
-						return 1;
 				}
 			}
 		}

sched_ctx_hypervisor/src/hypervisor_policies/policy_tools.h

@@ -17,8 +17,8 @@
 #include <sched_ctx_hypervisor.h>
 #include <pthread.h>
 
-#define HYPERVISOR_REDIM_SAMPLE 0.01
-#define HYPERVISOR_START_REDIM_SAMPLE 0.005
+#define HYPERVISOR_REDIM_SAMPLE 0.02
+#define HYPERVISOR_START_REDIM_SAMPLE 0.1
 
 struct bound_task_pool
 {

sched_ctx_hypervisor/src/sched_ctx_hypervisor.c

@@ -324,7 +324,7 @@ static void _get_cpus(int *workers, int nworkers, int *cpus, int *ncpus)
 	}
 }
 
-int get_nworkers_ctx(unsigned sched_ctx, enum starpu_archtype arch)
+int sched_ctx_hypervisor_get_nworkers_ctx(unsigned sched_ctx, enum starpu_archtype arch)
 {
 	int nworkers_ctx = 0;
 	struct starpu_sched_ctx_worker_collection *workers = starpu_sched_ctx_get_worker_collection(sched_ctx);
@@ -350,11 +350,11 @@ void sched_ctx_hypervisor_move_workers(unsigned sender_sched_ctx, unsigned recei
 {
 	if(nworkers_to_move > 0 && hypervisor.resize[sender_sched_ctx])// && hypervisor.resize[receiver_sched_ctx])
 	{
-/* 		int j; */
-/* 		printf("resize ctx %d with", sender_sched_ctx); */
-/* 		for(j = 0; j < nworkers_to_move; j++) */
-/* 			printf(" %d", workers_to_move[j]); */
-/* 		printf("\n"); */
+		int j;
+		printf("resize ctx %d with", sender_sched_ctx);
+		for(j = 0; j < nworkers_to_move; j++)
+			printf(" %d", workers_to_move[j]);
+		printf("\n");
 
 		int *cpus = (int*) malloc(nworkers_to_move * sizeof(int));
 		int ncpus;
@@ -368,11 +368,11 @@ void sched_ctx_hypervisor_move_workers(unsigned sender_sched_ctx, unsigned recei
 
 		if(now)
 		{
-/* 			int j; */
-/* 			printf("remove from ctx %d:", sender_sched_ctx); */
-/* 			for(j = 0; j < nworkers_to_move; j++) */
-/* 				printf(" %d", workers_to_move[j]); */
-/* 			printf("\n"); */
+			int j;
+			printf("remove now from ctx %d:", sender_sched_ctx);
+			for(j = 0; j < nworkers_to_move; j++)
+				printf(" %d", workers_to_move[j]);
+			printf("\n");
 
 			starpu_sched_ctx_remove_workers(workers_to_move, nworkers_to_move, sender_sched_ctx);
 		}
@@ -396,6 +396,7 @@ void sched_ctx_hypervisor_move_workers(unsigned sender_sched_ctx, unsigned recei
 				}
 
 				hypervisor.resize[sender_sched_ctx] = 0;
+//				hypervisor.resize[receiver_sched_ctx] = 0;
 
 				pthread_mutex_unlock(&hypervisor.sched_ctx_w[sender_sched_ctx].mutex);
 			}
@@ -413,11 +414,11 @@ void sched_ctx_hypervisor_add_workers_to_sched_ctx(int* workers_to_add, unsigned
 {
 	if(nworkers_to_add > 0 && hypervisor.resize[sched_ctx])
 	{
-/* 		int j; */
-/* 		printf("add to ctx %d:", sched_ctx); */
-/* 		for(j = 0; j < nworkers_to_add; j++) */
-/* 			printf(" %d", workers_to_add[j]); */
-/* 		printf("\n"); */
+		int j;
+		printf("add to ctx %d:", sched_ctx);
+		for(j = 0; j < nworkers_to_add; j++)
+			printf(" %d", workers_to_add[j]);
+		printf("\n");
 		starpu_sched_ctx_add_workers(workers_to_add, nworkers_to_add, sched_ctx);
 		struct sched_ctx_hypervisor_policy_config *new_config = sched_ctx_hypervisor_get_config(sched_ctx);
 		int i;
@@ -442,13 +443,13 @@ void sched_ctx_hypervisor_remove_workers_from_sched_ctx(int* workers_to_remove,
 
 		if(now)
 		{
-/* 				int j; */
-/* 				printf("remove from ctx %d:", sched_ctx); */
-/* 				for(j = 0; j < nworkers_to_remove; j++) */
-/* 					printf(" %d", workers_to_remove[j]); */
-/* 				printf("\n"); */
-
-				starpu_sched_ctx_remove_workers(workers_to_remove, nworkers_to_remove, sched_ctx);
+			int j;
+			printf("remove explicitley now from ctx %d:", sched_ctx);
+			for(j = 0; j < nworkers_to_remove; j++)
+				printf(" %d", workers_to_remove[j]);
+			printf("\n");
+			
+			starpu_sched_ctx_remove_workers(workers_to_remove, nworkers_to_remove, sched_ctx);
 		}
 		else
 		{
@@ -583,11 +584,11 @@ static unsigned _ack_resize_completed(unsigned sched_ctx, int worker)
 			   whatever the application says */
 			if(!((hypervisor.resize[sender_sched_ctx] == 0 || hypervisor.resize[receiver_sched_ctx] == 0) && imposed_resize))
 			{
-/* 				int j; */
-/* 				printf("remove from ctx %d:", sender_sched_ctx); */
-/* 				for(j = 0; j < nmoved_workers; j++) */
-/* 					printf(" %d", moved_workers[j]); */
-/* 				printf("\n"); */
+				int j;
+				printf("remove after ack from ctx %d:", sender_sched_ctx);
+				for(j = 0; j < nmoved_workers; j++)
+					printf(" %d", moved_workers[j]);
+				printf("\n");
 
 				starpu_sched_ctx_remove_workers(moved_workers, nmoved_workers, sender_sched_ctx);
 

src/core/sched_ctx.c

@@ -37,12 +37,19 @@ static void _get_workers_list(struct starpu_sched_ctx_worker_collection *workers
 
 static void _starpu_worker_gets_into_ctx(unsigned sched_ctx_id, struct _starpu_worker *worker)
 {
-	unsigned worker_sched_ctx_id = _starpu_worker_get_first_free_sched_ctx(worker);
-	struct _starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx_struct(sched_ctx_id);
-	/* add context to worker */
-	worker->sched_ctx[worker_sched_ctx_id] = sched_ctx;
-	worker->nsched_ctxs++;
-	worker->active_ctx = sched_ctx_id;
+	unsigned worker_sched_ctx_id = _starpu_worker_get_sched_ctx_id(worker, sched_ctx_id);
+	/* the worker was planning to go away in another ctx but finally he changed his mind & 
+	   he's staying */
+	if(worker_sched_ctx_id  == STARPU_NMAX_SCHED_CTXS)
+	{
+		unsigned worker_sched_ctx_id = _starpu_worker_get_first_free_sched_ctx(worker);
+		struct _starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx_struct(sched_ctx_id);
+		/* add context to worker */
+		worker->sched_ctx[worker_sched_ctx_id] = sched_ctx;
+		worker->nsched_ctxs++;
+		worker->active_ctx = sched_ctx_id;
+	}
+	worker->removed_from_ctx[sched_ctx_id] = 0;
 	return;
 }
 
@@ -58,55 +65,60 @@ void _starpu_worker_gets_out_of_ctx(unsigned sched_ctx_id, struct _starpu_worker
 	return;
 }
 
-static void _starpu_update_workers(int *workerids, int nworkers, int sched_ctx_id, unsigned now)
+static void _starpu_update_workers_with_ctx(int *workerids, int nworkers, int sched_ctx_id)
 {
 	int i;
-	struct _starpu_worker *worker[nworkers];
+	struct _starpu_worker *worker = NULL;
  	struct _starpu_worker *curr_worker = _starpu_get_local_worker_key();
-
+	
 	for(i = 0; i < nworkers; i++)
 	{
-		worker[i] = _starpu_get_worker_struct(workerids[i]);
+		worker = _starpu_get_worker_struct(workerids[i]);
+
+		/* if the current thread requires resize it's no need
+		   to lock it in order to change its  sched_ctx info */
+		if(curr_worker && curr_worker == worker)
+			_starpu_worker_gets_into_ctx(sched_ctx_id, worker);
+		else
+		{
+			_STARPU_PTHREAD_MUTEX_LOCK(&worker->sched_mutex);
+			_starpu_worker_gets_into_ctx(sched_ctx_id, worker);
+			_STARPU_PTHREAD_MUTEX_UNLOCK(&worker->sched_mutex);
+		}
+	}
 
+	return;
+}
 
-		unsigned worker_sched_ctx_id = _starpu_worker_get_sched_ctx_id(worker[i], sched_ctx_id);
-		/* if the worker is not in the ctx's list it means the update concerns the addition of ctxs*/
-		if(worker_sched_ctx_id == STARPU_NMAX_SCHED_CTXS)
+static void _starpu_update_workers_without_ctx(int *workerids, int nworkers, int sched_ctx_id, unsigned now)
+{
+	int i;
+	struct _starpu_worker *worker = NULL;
+ 	struct _starpu_worker *curr_worker = _starpu_get_local_worker_key();
+	
+	for(i = 0; i < nworkers; i++)
+	{
+		worker = _starpu_get_worker_struct(workerids[i]);
+		if(now)
 		{
-			/* if the current thread requires resize it's no need
-			   to lock it in order to change its  sched_ctx info */
-			if((curr_worker && curr_worker == worker[i]) || now)
-				_starpu_worker_gets_into_ctx(sched_ctx_id, worker[i]);
+			if(curr_worker && curr_worker == worker)
+				_starpu_worker_gets_out_of_ctx(sched_ctx_id, worker);
 			else
 			{
-				_STARPU_PTHREAD_MUTEX_LOCK(&worker[i]->sched_mutex);
-				_starpu_worker_gets_into_ctx(sched_ctx_id, worker[i]);
-				_STARPU_PTHREAD_MUTEX_UNLOCK(&worker[i]->sched_mutex);
+					_STARPU_PTHREAD_MUTEX_LOCK(&worker->sched_mutex);
+					_starpu_worker_gets_out_of_ctx(sched_ctx_id, worker);
+					_STARPU_PTHREAD_MUTEX_UNLOCK(&worker->sched_mutex);
 			}
 		}
-		else /*remove from context */
+		else
 		{
-			if(now)
-			{
-				if(curr_worker && curr_worker == worker[i])
-					_starpu_worker_gets_out_of_ctx(sched_ctx_id, worker[i]);
-				else
-				{
-					_STARPU_PTHREAD_MUTEX_LOCK(&worker[i]->sched_mutex);
-					_starpu_worker_gets_out_of_ctx(sched_ctx_id, worker[i]);
-					_STARPU_PTHREAD_MUTEX_UNLOCK(&worker[i]->sched_mutex);
-				}
-			}
+			if(curr_worker && curr_worker == worker)
+				worker->removed_from_ctx[sched_ctx_id] = 1;
 			else
 			{
-				if(curr_worker && curr_worker == worker[i])
-					worker[i]->removed_from_ctx[sched_ctx_id] = 1;
-				else
-				{
-					_STARPU_PTHREAD_MUTEX_LOCK(&worker[i]->sched_mutex);
-					worker[i]->removed_from_ctx[sched_ctx_id] = 1;
-					_STARPU_PTHREAD_MUTEX_UNLOCK(&worker[i]->sched_mutex);
-				}
+				_STARPU_PTHREAD_MUTEX_LOCK(&worker->sched_mutex);
+				worker->removed_from_ctx[sched_ctx_id] = 1;
+				_STARPU_PTHREAD_MUTEX_UNLOCK(&worker->sched_mutex);
 			}
 		}
 	}
@@ -138,6 +150,13 @@ static void _starpu_add_workers_to_sched_ctx(struct _starpu_sched_ctx *sched_ctx
 			int worker = workers->add(workers, (workerids == NULL ? i : workerids[i]));
 			if(worker >= 0)
 				added_workers[(*n_added_workers)++] = worker;
+			else
+			{
+				struct _starpu_worker *worker_str = _starpu_get_worker_struct(workerids[i]);
+				_STARPU_PTHREAD_MUTEX_LOCK(&worker_str->sched_mutex);
+				worker_str->removed_from_ctx[sched_ctx->id] = 0;
+				_STARPU_PTHREAD_MUTEX_UNLOCK(&worker_str->sched_mutex);
+			}
 		}
 		else
 		{
@@ -145,7 +164,7 @@ static void _starpu_add_workers_to_sched_ctx(struct _starpu_sched_ctx *sched_ctx
 			workers->add(workers, worker);
 			workers_to_add[i] = worker;
 		}
-	}
+}
 
 	if(sched_ctx->sched_policy->add_workers)
 	{
@@ -175,8 +194,6 @@ static void _starpu_remove_workers_from_sched_ctx(struct _starpu_sched_ctx *sche
 			if(worker >= 0)
 				removed_workers[(*n_removed_workers)++] = worker;
 		}
-/* 		if(*n_removed_workers) */
-/* 			sched_ctx->sched_policy->remove_workers(sched_ctx->id, removed_workers, *n_removed_workers); */
 	}
 
 	return;
@@ -403,7 +420,7 @@ unsigned starpu_sched_ctx_create_inside_interval(const char *policy_name, const
 	sched_ctx->min_ngpus = min_ngpus;
 	sched_ctx->max_ngpus = max_ngpus;
 
-	_starpu_update_workers(sched_ctx->workers->workerids, sched_ctx->workers->nworkers, sched_ctx->id, 0);
+	_starpu_update_workers_without_ctx(sched_ctx->workers->workerids, sched_ctx->workers->nworkers, sched_ctx->id, 0);
 #ifdef STARPU_USE_SCHED_CTX_HYPERVISOR
 	sched_ctx->perf_counters = NULL;
 #endif
@@ -416,7 +433,7 @@ unsigned starpu_sched_ctx_create(const char *policy_name, int *workerids,
 	struct _starpu_sched_ctx *sched_ctx = NULL;
 	sched_ctx = _starpu_create_sched_ctx(policy_name, workerids, nworkers, 0, sched_name);
 
-	_starpu_update_workers(sched_ctx->workers->workerids, sched_ctx->workers->nworkers, sched_ctx->id, 0);
+	_starpu_update_workers_with_ctx(sched_ctx->workers->workerids, sched_ctx->workers->nworkers, sched_ctx->id);
 #ifdef STARPU_USE_SCHED_CTX_HYPERVISOR
 	sched_ctx->perf_counters = NULL;
 #endif
@@ -480,8 +497,8 @@ void starpu_sched_ctx_delete(unsigned sched_ctx_id)
 		_STARPU_PTHREAD_MUTEX_LOCK(&changing_ctx_mutex[sched_ctx_id]);
 		/*if btw the mutex release & the mutex lock the context has changed take care to free all
 		  scheduling data before deleting the context */
-		_starpu_update_workers(workerids, nworkers_ctx, sched_ctx_id, 1);
-		_starpu_sched_ctx_free_scheduling_data(sched_ctx);
+		_starpu_update_workers_without_ctx(workerids, nworkers_ctx, sched_ctx_id, 1);
+//		_starpu_sched_ctx_free_scheduling_data(sched_ctx);
 		_starpu_delete_sched_ctx(sched_ctx);
 
 		_STARPU_PTHREAD_MUTEX_UNLOCK(&changing_ctx_mutex[sched_ctx_id]);
@@ -584,7 +601,7 @@ void starpu_sched_ctx_add_workers(int *workers_to_add, int nworkers_to_add, unsi
 		_starpu_add_workers_to_sched_ctx(sched_ctx, workers_to_add, nworkers_to_add, added_workers, &n_added_workers);
 		
 		if(n_added_workers > 0)
-			_starpu_update_workers(added_workers, n_added_workers, sched_ctx->id, 0);
+			_starpu_update_workers_with_ctx(added_workers, n_added_workers, sched_ctx->id);
 
 		_starpu_fetch_tasks_from_empty_ctx_list(sched_ctx);
 	}
@@ -610,7 +627,7 @@ void starpu_sched_ctx_remove_workers(int *workers_to_remove, int nworkers_to_rem
 		_starpu_remove_workers_from_sched_ctx(sched_ctx, workers_to_remove, nworkers_to_remove, removed_workers, &n_removed_workers);
 
 		if(n_removed_workers > 0)
-			_starpu_update_workers(removed_workers, n_removed_workers, sched_ctx->id, 0);
+			_starpu_update_workers_without_ctx(removed_workers, n_removed_workers, sched_ctx->id, 0);
 
 	}
 	_STARPU_PTHREAD_MUTEX_UNLOCK(&changing_ctx_mutex[sched_ctx_id]);

src/datawizard/memalloc.c

@@ -769,6 +769,11 @@ starpu_allocate_buffer_on_node(uint32_t dst_node, size_t size)
 {
 	uintptr_t addr = 0;
 
+	 
+#ifdef STARPU_USE_CUDA  
+	cudaError_t status;  
+#endif
+
 #ifdef STARPU_DEVEL
 #warning TODO: we need to use starpu_malloc
 #endif
@@ -792,7 +797,7 @@ starpu_allocate_buffer_on_node(uint32_t dst_node, size_t size)
 			addr = 1;
 			_STARPU_PTHREAD_MUTEX_UNLOCK(&cuda_alloc_mutex);
 #else
-			cudaError_t status = cudaMalloc((void **)&addr, size);
+			status = cudaMalloc((void **)&addr, size);
 			if (!addr || (status != cudaSuccess))
 			{
 				if (STARPU_UNLIKELY(status != cudaErrorMemoryAllocation))

src/sched_policies/deque_modeling_policy_data_aware.c

@@ -520,6 +520,7 @@ static void compute_all_performance_predictions(struct starpu_task *task,
 			_starpu_pthread_cond_t *sched_cond;
 			starpu_worker_get_sched_condition(worker, &sched_mutex, &sched_cond);
 
+			STARPU_ASSERT_MSG(fifo != NULL, "worker %d ctx %d\n", worker, sched_ctx_id);
 			_STARPU_PTHREAD_MUTEX_LOCK(sched_mutex);
 			fifo->exp_start = STARPU_MAX(fifo->exp_start, starpu_timing_now());
 			_STARPU_PTHREAD_MUTEX_UNLOCK(sched_mutex);
@@ -805,8 +806,11 @@ static void dmda_remove_workers(unsigned sched_ctx_id, int *workerids, unsigned
 	for (i = 0; i < nworkers; i++)
 	{
 		workerid = workerids[i];
-		_starpu_destroy_fifo(dt->queue_array[workerid]);
-		dt->queue_array[workerid] = NULL;
+		if(dt->queue_array[workerid] != NULL)
+		{
+			_starpu_destroy_fifo(dt->queue_array[workerid]);
+			dt->queue_array[workerid] = NULL;
+		}
 	}
 }