Merge remote-tracking branch 'origin/master' into ft_checkpoint

doc/doxygen/chapters/350_scheduling_policy_definition.doxy

@@ -60,6 +60,9 @@ queue the transfers on the idle prefetch queue, which is only processed when
 there are no non-idle prefetch to process.
 starpu_get_prefetch_flag() is a convenient helper for checking the value of the 
 \ref STARPU_PREFETCH environment variable.
+When a scheduler does such prefetching, it should set the <c>prefetches</c>
+field of the <c>starpu_sched_policy</c> to 1, to prevent the core from
+triggering its own prefetching.
 
 Usual functions can be used on tasks, for instance one can use the following to
 get the data size for a task.

include/starpu_scheduler.h

@@ -186,6 +186,11 @@ struct starpu_sched_policy
 	*/
 	void (*remove_workers)(unsigned sched_ctx_id, int *workerids, unsigned nworkers);
 
+	/** Whether this scheduling policy does data prefetching, and thus the
+	    core should not try to do it opportunistically.
+	*/
+	int prefetches;
+
 	/**
 	   Optional field. Name of the policy.
 	*/

src/core/sched_policy.c

@@ -576,22 +576,6 @@ int _starpu_push_task_to_workers(struct starpu_task *task)
 	{
 		struct _starpu_machine_config *config = _starpu_get_machine_config();
 
-		/* When a task can only be executed on a given arch and we have
-		 * only one memory node for that arch, we can systematically
-		 * prefetch before the scheduling decision. */
-		/* XXX: no, this is definitely not a good idea if the scheduler reorders tasks... */
-		if (0 && starpu_get_prefetch_flag() && starpu_memory_nodes_get_count() > 1)
-		{
-			if (task->where == STARPU_CPU && config->cpus_nodeid >= 0)
-				starpu_prefetch_task_input_on_node(task, config->cpus_nodeid);
-			else if (task->where == STARPU_CUDA && config->cuda_nodeid >= 0)
-				starpu_prefetch_task_input_on_node(task, config->cuda_nodeid);
-			else if (task->where == STARPU_OPENCL && config->opencl_nodeid >= 0)
-				starpu_prefetch_task_input_on_node(task, config->opencl_nodeid);
-			else if (task->where == STARPU_MIC && config->mic_nodeid >= 0)
-				starpu_prefetch_task_input_on_node(task, config->mic_nodeid);
-		}
-
 		if(!sched_ctx->sched_policy)
 		{
 			/* Note: we have to call that early, or else the task may have
@@ -634,6 +618,23 @@ int _starpu_push_task_to_workers(struct starpu_task *task)
 		}
 		else
 		{
+			/* When a task can only be executed on a given arch and we have
+			 * only one memory node for that arch, we can systematically
+			 * prefetch before the scheduling decision. */
+			if (!sched_ctx->sched_policy->prefetches
+				&& starpu_get_prefetch_flag()
+				&& starpu_memory_nodes_get_count() > 1)
+			{
+				if (task->where == STARPU_CPU && config->cpus_nodeid >= 0)
+					starpu_prefetch_task_input_on_node(task, config->cpus_nodeid);
+				else if (task->where == STARPU_CUDA && config->cuda_nodeid >= 0)
+					starpu_prefetch_task_input_on_node(task, config->cuda_nodeid);
+				else if (task->where == STARPU_OPENCL && config->opencl_nodeid >= 0)
+					starpu_prefetch_task_input_on_node(task, config->opencl_nodeid);
+				else if (task->where == STARPU_MIC && config->mic_nodeid >= 0)
+					starpu_prefetch_task_input_on_node(task, config->mic_nodeid);
+			}
+
 			STARPU_ASSERT(sched_ctx->sched_policy->push_task);
 			/* check out if there are any workers in the context */
 			unsigned nworkers = starpu_sched_ctx_get_nworkers(sched_ctx->id);

src/datawizard/coherency.c

@@ -743,7 +743,7 @@ int _starpu_fetch_data_on_node(starpu_data_handle_t handle, int node, struct _st
 	if (cpt == STARPU_SPIN_MAXTRY)
 		_starpu_spin_lock(&handle->header_lock);
 
-	if (is_prefetch > 0)
+	if (is_prefetch > STARPU_FETCH)
 	{
 		unsigned src_node_mask = 0;
 
@@ -761,7 +761,7 @@ int _starpu_fetch_data_on_node(starpu_data_handle_t handle, int node, struct _st
 		if (src_node_mask == 0)
 		{
 			/* no valid copy, nothing to prefetch */
-			_STARPU_DISP("Warning: no valid copy to prefetch?! that's not supposed to happen, please report\n");
+			STARPU_ASSERT_MSG(handle->init_cl, "Could not find a valid copy of the data, and no handle initialization function");
 			_starpu_spin_unlock(&handle->header_lock);
 			return 0;
 		}
@@ -915,7 +915,7 @@ int starpu_prefetch_task_input_on_node_prio(struct starpu_task *task, unsigned t
 	if (j->discontinuous != 0)
 		return 0;
 #endif
-	STARPU_ASSERT(!task->prefetched);
+	STARPU_ASSERT_MSG(!task->prefetched, "Prefetching was already requested for this task! Did you set 'prefetches' to 1 in the starpu_sched_policy structure?");
 	unsigned nbuffers = STARPU_TASK_GET_NBUFFERS(task);
 	unsigned index;
 
@@ -993,7 +993,7 @@ int starpu_prefetch_task_input_for_prio(struct starpu_task *task, unsigned worke
 	if (j->discontinuous != 0)
 		return 0;
 #endif
-	STARPU_ASSERT(!task->prefetched);
+	STARPU_ASSERT_MSG(!task->prefetched, "Prefetching was already requested for this task! Did you set 'prefetches' to 1 in the starpu_sched_policy structure?");
 	unsigned nbuffers = STARPU_TASK_GET_NBUFFERS(task);
 	unsigned index;
 
@@ -1250,15 +1250,19 @@ void _starpu_fetch_task_input_tail(struct starpu_task *task, struct _starpu_job
 		if (local_replicate->mc)
 		{
 			local_replicate->mc->diduse = 1;
-			if (task->prefetched &&
+			if (task->prefetched && local_replicate->initialized &&
+				/* See prefetch conditions in
+				 * starpu_prefetch_task_input_on_node_prio and alike */
 				!(mode & (STARPU_SCRATCH|STARPU_REDUX)) &&
 				(mode & STARPU_R))
 			{
 				/* Allocations or transfer prefetchs should have been done by now and marked
 				 * this mc as needed for us.
 				 * Now that we added a reference for the task, we can relieve that.  */
-				STARPU_ASSERT(local_replicate->mc->nb_tasks_prefetch > 0);
-				local_replicate->mc->nb_tasks_prefetch--;
+				/* Note: the replicate might have been evicted in between, thus not 100% sure
+				 * that our prefetch request is still recorded here.  */
+				if (local_replicate->mc->nb_tasks_prefetch > 0)
+					local_replicate->mc->nb_tasks_prefetch--;
 			}
 		}
 		_starpu_spin_unlock(&handle->header_lock);

src/sched_policies/deque_modeling_policy_data_aware.c

@@ -1191,6 +1191,7 @@ struct starpu_sched_policy _starpu_sched_dm_policy =
 	.policy_name = "dm",
 	.policy_description = "performance model",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };
 
 struct starpu_sched_policy _starpu_sched_dmda_policy =
@@ -1209,6 +1210,7 @@ struct starpu_sched_policy _starpu_sched_dmda_policy =
 	.policy_name = "dmda",
 	.policy_description = "data-aware performance model",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };
 
 struct starpu_sched_policy _starpu_sched_dmda_prio_policy =
@@ -1227,6 +1229,7 @@ struct starpu_sched_policy _starpu_sched_dmda_prio_policy =
 	.policy_name = "dmdap",
 	.policy_description = "data-aware performance model (priority)",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };
 
 struct starpu_sched_policy _starpu_sched_dmda_sorted_policy =
@@ -1245,6 +1248,7 @@ struct starpu_sched_policy _starpu_sched_dmda_sorted_policy =
 	.policy_name = "dmdas",
 	.policy_description = "data-aware performance model (sorted)",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };
 
 struct starpu_sched_policy _starpu_sched_dmda_sorted_decision_policy =
@@ -1263,6 +1267,7 @@ struct starpu_sched_policy _starpu_sched_dmda_sorted_decision_policy =
 	.policy_name = "dmdasd",
 	.policy_description = "data-aware performance model (sorted decision)",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };
 
 struct starpu_sched_policy _starpu_sched_dmda_ready_policy =
@@ -1281,4 +1286,5 @@ struct starpu_sched_policy _starpu_sched_dmda_ready_policy =
 	.policy_name = "dmdar",
 	.policy_description = "data-aware performance model (ready)",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/heteroprio.c

@@ -625,7 +625,8 @@ done:		;
 		     task_to_prefetch  = starpu_task_prio_list_next(&worker->tasks_queue.list, task_to_prefetch))
 		{
 			/* prefetch from closest to end task */
-			starpu_prefetch_task_input_for(task_to_prefetch, workerid);
+			if (!task_to_prefetch->prefetched) /* FIXME: it seems we are prefetching several times?? */
+				starpu_prefetch_task_input_for(task_to_prefetch, workerid);
 			nb_added_tasks -= 1;
 		}
 	}
@@ -649,4 +650,5 @@ struct starpu_sched_policy _starpu_sched_heteroprio_policy =
         .policy_name = "heteroprio",
         .policy_description = "heteroprio",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/hierarchical_heft.c

@@ -94,4 +94,5 @@ struct starpu_sched_policy _starpu_sched_tree_heft_hierarchical_policy =
 	.policy_name = "modular-heft-hierarchical",
 	.policy_description = "hierarchical heft tree policy",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/modular_eager_prefetching.c

@@ -48,4 +48,5 @@ struct starpu_sched_policy _starpu_sched_modular_eager_prefetching_policy =
 	.policy_name = "modular-eager-prefetching",
 	.policy_description = "eager with prefetching modular policy",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/modular_eager_prio.c

@@ -50,4 +50,5 @@ struct starpu_sched_policy _starpu_sched_modular_eager_prio_policy =
 	.policy_name = "modular-eager-prio",
 	.policy_description = "eager-prio modular policy",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/modular_gemm.c

@@ -199,4 +199,5 @@ struct starpu_sched_policy _starpu_sched_modular_gemm_policy =
 	.policy_name = "modular-gemm",
 	.policy_description = "gemm modular policy",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/modular_heft.c

@@ -81,4 +81,5 @@ struct starpu_sched_policy _starpu_sched_modular_heft_policy =
 	.policy_name = "modular-heft",
 	.policy_description = "heft modular policy",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/modular_heft2.c

@@ -81,4 +81,5 @@ struct starpu_sched_policy _starpu_sched_modular_heft2_policy =
 	.policy_name = "modular-heft2",
 	.policy_description = "heft modular2 policy",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/modular_heft_prio.c

@@ -82,4 +82,5 @@ struct starpu_sched_policy _starpu_sched_modular_heft_prio_policy =
 	.policy_name = "modular-heft-prio",
 	.policy_description = "heft+prio modular policy",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/modular_heteroprio.c

@@ -54,4 +54,5 @@ struct starpu_sched_policy _starpu_sched_modular_heteroprio_policy =
 	.policy_name = "modular-heteroprio",
 	.policy_description = "heteroprio modular policy",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/modular_heteroprio_heft.c

@@ -65,4 +65,5 @@ struct starpu_sched_policy _starpu_sched_modular_heteroprio_heft_policy =
 	.policy_name = "modular-heteroprio-heft",
 	.policy_description = "heteroprio+heft modular policy",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/modular_parallel_heft.c

@@ -85,4 +85,5 @@ struct starpu_sched_policy _starpu_sched_modular_parallel_heft_policy =
 	.policy_name = "modular-pheft",
 	.policy_description = "parallel heft modular policy",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/modular_parallel_random.c

@@ -86,4 +86,5 @@ struct starpu_sched_policy _starpu_sched_modular_parallel_random_prio_policy =
 	.policy_name = "modular-prandom-prio",
 	.policy_description = "prandom-prio modular policy",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/modular_prio_prefetching.c

@@ -90,4 +90,5 @@ struct starpu_sched_policy _starpu_sched_modular_prio_prefetching_policy =
 	.policy_name = "modular-prio-prefetching",
 	.policy_description = "prio prefetching modular policy",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/modular_random_prefetching.c

@@ -90,4 +90,5 @@ struct starpu_sched_policy _starpu_sched_modular_random_prio_prefetching_policy
 	.policy_name = "modular-random-prio-prefetching",
 	.policy_description = "random-prio prefetching modular policy",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };

src/sched_policies/parallel_heft.c

@@ -605,4 +605,5 @@ struct starpu_sched_policy _starpu_sched_parallel_heft_policy =
 	.policy_name = "pheft",
 	.policy_description = "parallel HEFT",
 	.worker_type = STARPU_WORKER_LIST,
+	.prefetches = 1,
 };