Add 'ready' heuristic to modular schedulers

ChangeLog

@@ -42,6 +42,7 @@ New features:
     fields of starpu_task to 0.
   * starpufft: Support 3D.
   * New modular-eager-prio scheduler.
+  * Add 'ready' heuristic to modular schedulers.
 
 Changes:
   * Modification in the Native Fortran interface of the functions

doc/doxygen/chapters/501_environment_variables.doxy

@@ -512,6 +512,15 @@ the coefficient to be applied to it before adding it to the computation part.
 Define the execution time penalty of a joule (\ref Energy-basedScheduling).
 </dd>
 
+<dt>STARPU_SCHED_READY</dt>
+<dd>
+\anchor STARPU_SCHED_READY
+\addindex __env__STARPU_SCHED_READY
+For a modular scheduler with sorted queues below the decision component, workers
+pick up a task which has most of its data already available. Setting this to 0
+disables this.
+</dd>
+
 <dt>STARPU_IDLE_POWER</dt>
 <dd>
 \anchor STARPU_IDLE_POWER

include/starpu_sched_component.h

@@ -416,6 +416,7 @@ struct starpu_sched_component_fifo_data
 {
 	unsigned ntasks_threshold;
 	double exp_len_threshold;
+	int ready;
 };
 
 /**
@@ -441,6 +442,7 @@ struct starpu_sched_component_prio_data
 {
 	unsigned ntasks_threshold;
 	double exp_len_threshold;
+	int ready;
 };
 struct starpu_sched_component *starpu_sched_component_prio_create(struct starpu_sched_tree *tree, struct starpu_sched_component_prio_data *prio_data) STARPU_ATTRIBUTE_MALLOC;
 int starpu_sched_component_is_prio(struct starpu_sched_component *component);
@@ -721,14 +723,19 @@ struct starpu_sched_tree *starpu_sched_component_make_scheduler(unsigned sched_c
 #define STARPU_SCHED_SIMPLE_FIFOS_BELOW_PRIO	(1<<9)
 
 /**
+   Request that the fifos below be pulled rather ready tasks
+*/
+#define STARPU_SCHED_SIMPLE_FIFOS_BELOW_READY	(1<<10)
+
+/**
    Request that work between workers using the same fifo below be distributed using a work stealing component.
 */
-#define STARPU_SCHED_SIMPLE_WS_BELOW		(1<<10)
+#define STARPU_SCHED_SIMPLE_WS_BELOW		(1<<11)
 
 /**
    Request to not only choose between simple workers, but also choose between combined workers.
 */
-#define STARPU_SCHED_SIMPLE_COMBINED_WORKERS	(1<<11)
+#define STARPU_SCHED_SIMPLE_COMBINED_WORKERS	(1<<12)
 
 /**
    Create a simple modular scheduler tree around a scheduling decision-making

src/sched_policies/component_fifo.c

@@ -29,6 +29,7 @@ struct _starpu_fifo_data
 	starpu_pthread_mutex_t mutex;
 	unsigned ntasks_threshold;
 	double exp_len_threshold;
+	int ready;
 };
 
 static void fifo_component_deinit_data(struct starpu_sched_component * component)
@@ -154,7 +155,7 @@ static int fifo_push_task(struct starpu_sched_component * component, struct star
 	return fifo_push_local_task(component, task, 0);
 }
 
-static struct starpu_task * fifo_pull_task(struct starpu_sched_component * component, struct starpu_sched_component * to STARPU_ATTRIBUTE_UNUSED)
+static struct starpu_task * fifo_pull_task(struct starpu_sched_component * component, struct starpu_sched_component * to)
 {
 	STARPU_ASSERT(component && component->data);
 	struct _starpu_fifo_data * data = component->data;
@@ -162,7 +163,11 @@ static struct starpu_task * fifo_pull_task(struct starpu_sched_component * compo
 	starpu_pthread_mutex_t * mutex = &data->mutex;
 	const double now = starpu_timing_now();
 	STARPU_COMPONENT_MUTEX_LOCK(mutex);
-	struct starpu_task * task = _starpu_fifo_pop_task(fifo, starpu_worker_get_id_check());
+	struct starpu_task * task;
+	if (data->ready && to->properties & STARPU_SCHED_COMPONENT_SINGLE_MEMORY_NODE)
+		task = _starpu_fifo_pop_first_ready_task(fifo, starpu_bitmap_first(to->workers_in_ctx), -1);
+	else
+		task = _starpu_fifo_pop_task(fifo, starpu_worker_get_id_check());
 	if(task)
 	{
 		if(!isnan(task->predicted))
@@ -261,11 +266,13 @@ struct starpu_sched_component * starpu_sched_component_fifo_create(struct starpu
 	{
 		data->ntasks_threshold=params->ntasks_threshold;
 		data->exp_len_threshold=params->exp_len_threshold;
+		data->ready=params->ready;
 	}
 	else
 	{
 		data->ntasks_threshold=0;
 		data->exp_len_threshold=0.0;
+		data->ready=0;
 	}
 
 	return component;

src/sched_policies/component_prio.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2013,2017                                Inria
  * Copyright (C) 2014-2017                                CNRS
- * Copyright (C) 2014,2015,2017,2018                      Université de Bordeaux
+ * Copyright (C) 2014,2015,2017,2018-2019                 Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -48,6 +48,7 @@ struct _starpu_prio_data
 	starpu_pthread_mutex_t mutex;
 	unsigned ntasks_threshold;
 	double exp_len_threshold;
+	int ready;
 };
 
 static void prio_component_deinit_data(struct starpu_sched_component * component)
@@ -174,7 +175,7 @@ static int prio_push_task(struct starpu_sched_component * component, struct star
 	return ret;
 }
 
-static struct starpu_task * prio_pull_task(struct starpu_sched_component * component, struct starpu_sched_component * to STARPU_ATTRIBUTE_UNUSED)
+static struct starpu_task * prio_pull_task(struct starpu_sched_component * component, struct starpu_sched_component * to)
 {
 	STARPU_ASSERT(component && component->data);
 	struct _starpu_prio_data * data = component->data;
@@ -182,7 +183,11 @@ static struct starpu_task * prio_pull_task(struct starpu_sched_component * compo
 	starpu_pthread_mutex_t * mutex = &data->mutex;
 	const double now = starpu_timing_now();
 	STARPU_COMPONENT_MUTEX_LOCK(mutex);
-	struct starpu_task * task = _starpu_prio_deque_pop_task(prio);
+	struct starpu_task * task;
+	if (data->ready && to->properties & STARPU_SCHED_COMPONENT_SINGLE_MEMORY_NODE)
+		task = _starpu_prio_deque_deque_first_ready_task(prio, starpu_bitmap_first(to->workers_in_ctx));
+	else
+		task = _starpu_prio_deque_pop_task(prio);
 	if(task)
 	{
 		if(!isnan(task->predicted))
@@ -282,11 +287,13 @@ struct starpu_sched_component * starpu_sched_component_prio_create(struct starpu
 	{
 		data->ntasks_threshold=params->ntasks_threshold;
 		data->exp_len_threshold=params->exp_len_threshold;
+		data->ready=params->ready;
 	}
 	else
 	{
 		data->ntasks_threshold=0;
 		data->exp_len_threshold=0.0;
+		data->ready=0;
 	}
 
 	return component;

src/sched_policies/fifo_queues.c

@@ -26,6 +26,7 @@
 #include <sched_policies/fifo_queues.h>
 #include <common/fxt.h>
 #include <core/topology.h>
+#include <limits.h>
 /*
 static int is_sorted_task_list(struct starpu_task * task)
 {

src/sched_policies/modular_eager_prefetching.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2013-2015,2017                           Inria
  * Copyright (C) 2014,2017                                CNRS
- * Copyright (C) 2014,2016-2018                           Université de Bordeaux
+ * Copyright (C) 2014,2016-2019                           Université de Bordeaux
  * Copyright (C) 2013                                     Simon Archipoff
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -26,6 +26,7 @@ static void initialize_eager_prefetching_center_policy(unsigned sched_ctx_id)
 			STARPU_SCHED_SIMPLE_DECIDE_WORKERS |
 			STARPU_SCHED_SIMPLE_FIFO_ABOVE |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW |
+			STARPU_SCHED_SIMPLE_FIFOS_BELOW_READY |
 			STARPU_SCHED_SIMPLE_IMPL, sched_ctx_id);
 }
 

src/sched_policies/modular_eager_prio.c

@@ -28,6 +28,7 @@ static void initialize_eager_prio_center_policy(unsigned sched_ctx_id)
 			STARPU_SCHED_SIMPLE_DECIDE_WORKERS |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW_PRIO |
+			STARPU_SCHED_SIMPLE_FIFOS_BELOW_READY |
 			STARPU_SCHED_SIMPLE_IMPL, sched_ctx_id);
 }
 

src/sched_policies/modular_ez.c

@@ -213,16 +213,20 @@ void starpu_sched_component_initialize_simple_scheduler(starpu_sched_component_c
 	double exp_len_threshold = _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT;
 	exp_len_threshold = starpu_get_env_float_default("STARPU_EXP_LEN_THRESHOLD", exp_len_threshold);
 
+	int ready = starpu_get_env_number_default("STARPU_SCHED_READY", flags & STARPU_SCHED_SIMPLE_FIFOS_BELOW_READY ? 1 : 0);
+
 	struct starpu_sched_component_prio_data prio_data =
 		{
 			.ntasks_threshold = ntasks_threshold,
 			.exp_len_threshold = exp_len_threshold,
+			.ready = ready,
 		};
 
 	struct starpu_sched_component_fifo_data fifo_data =
 		{
 			.ntasks_threshold = ntasks_threshold,
 			.exp_len_threshold = exp_len_threshold,
+			.ready = ready,
 		};
 
 

src/sched_policies/modular_heft.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2013-2015,2017                           Inria
  * Copyright (C) 2014,2015,2017                           CNRS
- * Copyright (C) 2013-2015,2017,2018                      Université de Bordeaux
+ * Copyright (C) 2013-2015,2017,2018-2019                 Université de Bordeaux
  * Copyright (C) 2013                                     Simon Archipoff
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -58,6 +58,7 @@ static void initialize_heft_center_policy(unsigned sched_ctx_id)
 			STARPU_SCHED_SIMPLE_FIFO_ABOVE_PRIO |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW_PRIO |
+			STARPU_SCHED_SIMPLE_FIFOS_BELOW_READY |
 			STARPU_SCHED_SIMPLE_IMPL, sched_ctx_id);
 }
 

src/sched_policies/modular_heft2.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2013-2015,2017                           Inria
  * Copyright (C) 2014,2015,2017                           CNRS
- * Copyright (C) 2013-2015,2017,2018                      Université de Bordeaux
+ * Copyright (C) 2013-2015,2017,2018-2019                 Université de Bordeaux
  * Copyright (C) 2013                                     Simon Archipoff
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -58,6 +58,7 @@ static void initialize_heft2_center_policy(unsigned sched_ctx_id)
 			STARPU_SCHED_SIMPLE_FIFO_ABOVE_PRIO |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW_PRIO |
+			STARPU_SCHED_SIMPLE_FIFOS_BELOW_READY |
 			STARPU_SCHED_SIMPLE_IMPL, sched_ctx_id);
 }
 

src/sched_policies/modular_heft_prio.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2014,2015,2017                           CNRS
  * Copyright (C) 2013-2015,2017                           Inria
- * Copyright (C) 2013-2015,2017,2018                      Université de Bordeaux
+ * Copyright (C) 2013-2015,2017,2018-2019                 Université de Bordeaux
  * Copyright (C) 2013                                     Simon Archipoff
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -59,6 +59,7 @@ static void initialize_heft_prio_policy(unsigned sched_ctx_id)
 			STARPU_SCHED_SIMPLE_FIFO_ABOVE_PRIO |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW_PRIO |
+			STARPU_SCHED_SIMPLE_FIFOS_BELOW_READY |
 			STARPU_SCHED_SIMPLE_IMPL, sched_ctx_id);
 }
 

src/sched_policies/modular_parallel_heft.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2013-2015,2017                           Inria
  * Copyright (C) 2014,2015,2017                           CNRS
- * Copyright (C) 2013-2015,2017,2018-2019                      Université de Bordeaux
+ * Copyright (C) 2013-2015,2017,2018-2019                 Université de Bordeaux
  * Copyright (C) 2013                                     Simon Archipoff
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -60,6 +60,7 @@ static void initialize_parallel_heft_center_policy(unsigned sched_ctx_id)
 			STARPU_SCHED_SIMPLE_FIFO_ABOVE_PRIO |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW_PRIO |
+			STARPU_SCHED_SIMPLE_FIFOS_BELOW_READY |
 			STARPU_SCHED_SIMPLE_IMPL, sched_ctx_id);
 }
 

src/sched_policies/modular_prio_prefetching.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2013-2015,2017                           Inria
  * Copyright (C) 2014,2017                                CNRS
- * Copyright (C) 2014,2017,2018                           Université de Bordeaux
+ * Copyright (C) 2014,2017,2018-2019                      Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -67,6 +67,8 @@ static void initialize_prio_prefetching_center_policy(unsigned sched_ctx_id)
 			STARPU_SCHED_SIMPLE_FIFO_ABOVE |
 			STARPU_SCHED_SIMPLE_FIFO_ABOVE_PRIO |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW |
+			STARPU_SCHED_SIMPLE_FIFOS_BELOW_PRIO |
+			STARPU_SCHED_SIMPLE_FIFOS_BELOW_READY |
 			STARPU_SCHED_SIMPLE_IMPL, sched_ctx_id);
 }
 

src/sched_policies/modular_random_prefetching.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2013-2015,2017                           Inria
  * Copyright (C) 2014,2017                                CNRS
- * Copyright (C) 2014,2017,2018                           Université de Bordeaux
+ * Copyright (C) 2014,2017,2018-2019                      Université de Bordeaux
  * Copyright (C) 2013                                     Simon Archipoff
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -32,6 +32,7 @@ static void initialize_random_fifo_prefetching_center_policy(unsigned sched_ctx_
 			STARPU_SCHED_SIMPLE_DECIDE_WORKERS |
 			STARPU_SCHED_SIMPLE_FIFO_ABOVE |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW |
+			STARPU_SCHED_SIMPLE_FIFOS_BELOW_READY |
 			STARPU_SCHED_SIMPLE_IMPL, sched_ctx_id);
 }
 
@@ -67,6 +68,7 @@ static void initialize_random_prio_prefetching_center_policy(unsigned sched_ctx_
 			STARPU_SCHED_SIMPLE_FIFO_ABOVE_PRIO |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW |
 			STARPU_SCHED_SIMPLE_FIFOS_BELOW_PRIO |
+			STARPU_SCHED_SIMPLE_FIFOS_BELOW_READY |
 			STARPU_SCHED_SIMPLE_IMPL, sched_ctx_id);
 }
 

src/sched_policies/prio_deque.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2013,2014                                Inria
  * Copyright (C) 2016,2017                                CNRS
- * Copyright (C) 2014,2017                                Université de Bordeaux
+ * Copyright (C) 2014,2017, 2019                                Université de Bordeaux
  * Copyright (C) 2013                                     Simon Archipoff
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -20,6 +20,7 @@
 #include <core/workers.h>
 
 #include "prio_deque.h"
+#include "fifo_queues.h"
 
 
 /* a little dirty code factorization */
@@ -78,3 +79,48 @@ struct starpu_task * _starpu_prio_deque_deque_task_for_worker(struct _starpu_pri
 	STARPU_ASSERT(workerid >= 0 && (unsigned) workerid < starpu_worker_get_count());
 	REMOVE_TASK(pdeque, _tail, next, pred_can_execute, &workerid);
 }
+
+struct starpu_task *_starpu_prio_deque_deque_first_ready_task(struct _starpu_prio_deque * pdeque, unsigned workerid)
+{
+	struct starpu_task *task = NULL, *current;
+
+	if (starpu_task_prio_list_empty(&pdeque->list))
+		return NULL;
+
+	if (pdeque->ntasks > 0)
+	{
+		pdeque->ntasks--;
+
+		task = starpu_task_prio_list_front_highest(&pdeque->list);
+		if (STARPU_UNLIKELY(!task))
+			return NULL;
+
+		int first_task_priority = task->priority;
+		int non_ready_best = INT_MAX;
+
+		for (current = starpu_task_prio_list_begin(&pdeque->list);
+		     current != starpu_task_prio_list_end(&pdeque->list);
+		     current = starpu_task_prio_list_next(&pdeque->list, current))
+		{
+			int priority = current->priority;
+
+			if (priority >= first_task_priority)
+			{
+				int non_ready = _starpu_count_non_ready_buffers(current, workerid);
+				if (non_ready < non_ready_best)
+				{
+					non_ready_best = non_ready;
+					task = current;
+
+					if (non_ready == 0)
+						break;
+				}
+			}
+		}
+
+		starpu_task_prio_list_erase(&pdeque->list, task);
+	}
+
+	return task;
+}
+

src/sched_policies/prio_deque.h

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2013,2018                                Inria
  * Copyright (C) 2017                                     CNRS
- * Copyright (C) 2014,2017,2018                           Université de Bordeaux
+ * Copyright (C) 2014,2017,2018-2019                      Université de Bordeaux
  * Copyright (C) 2013                                     Simon Archipoff
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -134,4 +134,6 @@ static inline struct starpu_task * _starpu_prio_deque_deque_task(struct _starpu_
  */
 struct starpu_task * _starpu_prio_deque_deque_task_for_worker(struct _starpu_prio_deque *, int workerid, int *skipped);
 
+struct starpu_task *_starpu_prio_deque_deque_first_ready_task(struct _starpu_prio_deque *, unsigned workerid);
+
 #endif /* __PRIO_DEQUE_H__ */